1 /* Licensed under LGPLv2.1+ - see LICENSE file for details */
11 #include <ccan/build_assert/build_assert.h>
12 #include <ccan/likely/likely.h>
13 #include <ccan/alignof/alignof.h>
14 #include <ccan/short_types/short_types.h>
15 #include <ccan/compiler/compiler.h>
19 Inspired by (and parts taken from) Andrew Tridgell's alloc_mmap:
20 http://samba.org/~tridge/junkcode/alloc_mmap/
22 Copyright (C) Andrew Tridgell 2007
24 This library is free software; you can redistribute it and/or
25 modify it under the terms of the GNU Lesser General Public
26 License as published by the Free Software Foundation; either
27 version 2 of the License, or (at your option) any later version.
29 This library is distributed in the hope that it will be useful,
30 but WITHOUT ANY WARRANTY; without even the implied warranty of
31 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
32 Lesser General Public License for more details.
34 You should have received a copy of the GNU Lesser General Public
35 License along with this library; if not, write to the Free Software
36 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
39 /* We divide the pool into this many large pages (nearest power of 2) */
40 #define MAX_LARGE_PAGES (256UL)
42 /* 32 small pages == 1 large page. */
43 #define BITS_FROM_SMALL_TO_LARGE_PAGE 5
45 #define MAX_SMALL_PAGES (MAX_LARGE_PAGES << BITS_FROM_SMALL_TO_LARGE_PAGE)
47 /* Smallest pool size for this scheme: 128-byte small pages. That's
48 * 9/13% overhead for 32/64 bit. */
49 #define MIN_USEFUL_SIZE (MAX_SMALL_PAGES * 128)
51 /* Every 4 buckets, we jump up a power of 2. ...8 10 12 14 16 20 24 28 32... */
52 #define INTER_BUCKET_SPACE 4
54 #define SMALL_PAGES_PER_LARGE_PAGE (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)
56 /* FIXME: Figure this out properly. */
57 #define MAX_SIZE (1 << 30)
59 /* How few object to fit in a page before using a larger one? (8) */
60 #define MAX_PAGE_OBJECT_ORDER 3
62 #define BITS_PER_LONG (sizeof(long) * CHAR_BIT)
65 u32 elements_per_page;
71 /* Bitmap of which pages are large. */
72 unsigned long pagesize[MAX_LARGE_PAGES / BITS_PER_LONG];
74 /* List of unused small/large pages. */
78 /* List of huge allocs. */
81 /* This is less defined: we have two buckets for each power of 2 */
82 struct bucket_state bs[1];
86 unsigned long next, prev;
87 unsigned long off, len;
92 /* FIXME: We can just count all-0 and all-1 used[] elements. */
93 unsigned elements_used : 25;
95 unsigned long used[1]; /* One bit per element. */
99 * Every 4 buckets, the size doubles.
100 * Between buckets, sizes increase linearly.
102 * eg. bucket 40 = 2^10 = 1024
103 * bucket 41 = 2^10 + 2^10*4 = 1024 + 256
104 * bucket 42 = 2^10 + 2^10*4 = 1024 + 512
105 * bucket 43 = 2^10 + 2^10*4 = 1024 + 768
106 * bucket 45 = 2^11 = 2048
108 * Care is taken to handle low numbered buckets, at cost of overflow.
110 static unsigned long bucket_to_size(unsigned int bucket)
112 unsigned long base = 1UL << (bucket / INTER_BUCKET_SPACE);
113 return base + ((bucket % INTER_BUCKET_SPACE)
114 << (bucket / INTER_BUCKET_SPACE))
115 / INTER_BUCKET_SPACE;
120 * fls(size/2) == 3. 1 << 3 == 8, so we're 2 too large, out of a possible
121 * 8 too large. That's 1/4 of the way to the next power of 2 == 1 bucket.
123 * We make sure we round up. Note that this fails on 32 bit at size
124 * 1879048193 (around bucket 120).
126 static unsigned int size_to_bucket(unsigned long size)
128 unsigned int base = afls(size/2);
129 unsigned long overshoot;
131 overshoot = size - (1UL << base);
132 return base * INTER_BUCKET_SPACE
133 + ((overshoot * INTER_BUCKET_SPACE + (1UL << base)-1) >> base);
136 static unsigned int small_page_bits(unsigned long poolsize)
138 return afls(poolsize / MAX_SMALL_PAGES - 1);
141 static struct page_header *from_pgnum(struct header *head,
145 return (struct page_header *)((char *)head + (pgnum << sp_bits));
148 static u16 to_pgnum(struct header *head, void *p, unsigned sp_bits)
150 return ((char *)p - (char *)head) >> sp_bits;
153 static size_t used_size(unsigned int num_elements)
155 return align_up(num_elements, BITS_PER_LONG) / CHAR_BIT;
159 * We always align the first entry to the lower power of 2.
160 * eg. the 12-byte bucket gets 8-byte aligned. The 4096-byte bucket
161 * gets 4096-byte aligned.
163 static unsigned long page_header_size(unsigned int align_bits,
164 unsigned long num_elements)
168 size = sizeof(struct page_header)
169 - sizeof(((struct page_header *)0)->used)
170 + used_size(num_elements);
171 return align_up(size, 1UL << align_bits);
174 static void add_to_list(struct header *head,
175 u16 *list, struct page_header *ph, unsigned sp_bits)
177 unsigned long h = *list, offset = to_pgnum(head, ph, sp_bits);
181 struct page_header *prev = from_pgnum(head, h, sp_bits);
182 assert(prev->prev == 0);
189 static void del_from_list(struct header *head,
190 u16 *list, struct page_header *ph, unsigned sp_bits)
196 struct page_header *prev = from_pgnum(head, ph->prev, sp_bits);
197 prev->next = ph->next;
200 struct page_header *next = from_pgnum(head, ph->next, sp_bits);
201 next->prev = ph->prev;
205 static u16 pop_from_list(struct header *head,
207 unsigned int sp_bits)
210 struct page_header *ph = from_pgnum(head, h, sp_bits);
215 from_pgnum(head, *list, sp_bits)->prev = 0;
220 static void add_to_huge_list(struct header *head, struct huge_alloc *ha)
222 unsigned long h = head->huge;
223 unsigned long offset = (char *)ha - (char *)head;
227 struct huge_alloc *prev = (void *)((char *)head + h);
228 assert(prev->prev == 0);
235 static void del_from_huge(struct header *head, struct huge_alloc *ha)
239 head->huge = ha->next;
241 struct huge_alloc *prev = (void *)((char *)head + ha->prev);
242 prev->next = ha->next;
245 struct huge_alloc *next = (void *)((char *)head + ha->next);
246 next->prev = ha->prev;
250 static void add_small_page_to_freelist(struct header *head,
251 struct page_header *ph,
252 unsigned int sp_bits)
254 add_to_list(head, &head->small_free_list, ph, sp_bits);
257 static void add_large_page_to_freelist(struct header *head,
258 struct page_header *ph,
259 unsigned int sp_bits)
261 add_to_list(head, &head->large_free_list, ph, sp_bits);
264 static void add_to_bucket_list(struct header *head,
265 struct bucket_state *bs,
266 struct page_header *ph,
267 unsigned int sp_bits)
269 add_to_list(head, &bs->page_list, ph, sp_bits);
272 static void del_from_bucket_list(struct header *head,
273 struct bucket_state *bs,
274 struct page_header *ph,
275 unsigned int sp_bits)
277 del_from_list(head, &bs->page_list, ph, sp_bits);
280 static void del_from_bucket_full_list(struct header *head,
281 struct bucket_state *bs,
282 struct page_header *ph,
283 unsigned int sp_bits)
285 del_from_list(head, &bs->full_list, ph, sp_bits);
288 static void add_to_bucket_full_list(struct header *head,
289 struct bucket_state *bs,
290 struct page_header *ph,
291 unsigned int sp_bits)
293 add_to_list(head, &bs->full_list, ph, sp_bits);
296 static void clear_bit(unsigned long bitmap[], unsigned int off)
298 bitmap[off / BITS_PER_LONG] &= ~(1UL << (off % BITS_PER_LONG));
301 static bool test_bit(const unsigned long bitmap[], unsigned int off)
303 return bitmap[off / BITS_PER_LONG] & (1UL << (off % BITS_PER_LONG));
306 static void set_bit(unsigned long bitmap[], unsigned int off)
308 bitmap[off / BITS_PER_LONG] |= (1UL << (off % BITS_PER_LONG));
311 /* There must be a bit to be found. */
312 static unsigned int find_free_bit(const unsigned long bitmap[])
316 for (i = 0; bitmap[i] == -1UL; i++);
317 return (i*BITS_PER_LONG) + affsl(~bitmap[i]) - 1;
320 /* How many elements can we fit in a page? */
321 static unsigned long elements_per_page(unsigned long align_bits,
325 unsigned long num, overhead;
327 /* First approximation: no extra room for bitmap. */
328 overhead = align_up(sizeof(struct page_header), 1UL << align_bits);
329 num = (psize - overhead) / esize;
331 while (page_header_size(align_bits, num) + esize * num > psize)
336 static bool large_page_bucket(unsigned int bucket, unsigned int sp_bits)
338 unsigned long max_smallsize;
340 /* Note: this doesn't take into account page header. */
341 max_smallsize = (1UL << sp_bits) >> MAX_PAGE_OBJECT_ORDER;
343 return bucket_to_size(bucket) > max_smallsize;
346 static unsigned int max_bucket(unsigned int lp_bits)
348 return (lp_bits - MAX_PAGE_OBJECT_ORDER) * INTER_BUCKET_SPACE;
351 void alloc_init(void *pool, unsigned long poolsize)
353 struct header *head = pool;
354 struct page_header *ph;
355 unsigned int lp_bits, sp_bits, num_buckets;
356 unsigned long header_size, i;
358 if (poolsize < MIN_USEFUL_SIZE) {
359 tiny_alloc_init(pool, poolsize);
363 /* We rely on page numbers fitting in 16 bit. */
364 BUILD_ASSERT(MAX_SMALL_PAGES < 65536);
366 sp_bits = small_page_bits(poolsize);
367 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
369 num_buckets = max_bucket(lp_bits);
372 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
374 memset(head, 0, header_size);
375 for (i = 0; i < num_buckets; i++) {
376 unsigned long pagesize;
378 if (large_page_bucket(i, sp_bits))
379 pagesize = 1UL << lp_bits;
381 pagesize = 1UL << sp_bits;
383 head->bs[i].elements_per_page
384 = elements_per_page(i / INTER_BUCKET_SPACE,
389 /* They start as all large pages. */
390 memset(head->pagesize, 0xFF, sizeof(head->pagesize));
391 /* FIXME: small pages for last bit? */
393 /* Split first page into small pages. */
394 assert(header_size < (1UL << lp_bits));
395 clear_bit(head->pagesize, 0);
397 /* Skip over page(s) used by header, add rest to free list */
398 for (i = align_up(header_size, (1UL << sp_bits)) >> sp_bits;
399 i < SMALL_PAGES_PER_LARGE_PAGE;
401 ph = from_pgnum(head, i, sp_bits);
402 ph->elements_used = 0;
403 add_small_page_to_freelist(head, ph, sp_bits);
406 /* Add the rest of the pages as large pages. */
407 i = SMALL_PAGES_PER_LARGE_PAGE;
408 while ((i << sp_bits) + (1UL << lp_bits) <= poolsize) {
409 assert(i < MAX_SMALL_PAGES);
410 ph = from_pgnum(head, i, sp_bits);
411 ph->elements_used = 0;
412 add_large_page_to_freelist(head, ph, sp_bits);
413 i += SMALL_PAGES_PER_LARGE_PAGE;
417 /* A large page worth of small pages are free: delete them from free list. */
418 static void del_large_from_small_free_list(struct header *head,
419 struct page_header *ph,
420 unsigned int sp_bits)
424 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
425 del_from_list(head, &head->small_free_list,
426 (struct page_header *)((char *)ph
432 static bool all_empty(struct header *head,
438 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
439 struct page_header *ph = from_pgnum(head, pgnum + i, sp_bits);
440 if (ph->elements_used)
446 static void recombine_small_pages(struct header *head, unsigned long poolsize,
447 unsigned int sp_bits)
450 unsigned int lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
452 /* Look for small pages to coalesce, after first large page. */
453 for (i = SMALL_PAGES_PER_LARGE_PAGE;
454 i < (poolsize >> lp_bits) << BITS_FROM_SMALL_TO_LARGE_PAGE;
455 i += SMALL_PAGES_PER_LARGE_PAGE) {
456 /* Already a large page? */
457 if (test_bit(head->pagesize, i / SMALL_PAGES_PER_LARGE_PAGE))
459 if (all_empty(head, i, sp_bits)) {
460 struct page_header *ph = from_pgnum(head, i, sp_bits);
461 set_bit(head->pagesize,
462 i / SMALL_PAGES_PER_LARGE_PAGE);
463 del_large_from_small_free_list(head, ph, sp_bits);
464 add_large_page_to_freelist(head, ph, sp_bits);
469 static u16 get_large_page(struct header *head, unsigned long poolsize,
470 unsigned int sp_bits)
474 page = pop_from_list(head, &head->large_free_list, sp_bits);
478 recombine_small_pages(head, poolsize, sp_bits);
480 return pop_from_list(head, &head->large_free_list, sp_bits);
483 /* Returns small page. */
484 static unsigned long break_up_large_page(struct header *head,
485 unsigned int sp_bits,
490 clear_bit(head->pagesize, lpage >> BITS_FROM_SMALL_TO_LARGE_PAGE);
492 for (i = 1; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
493 struct page_header *ph = from_pgnum(head, lpage + i, sp_bits);
494 /* Initialize this: huge_alloc reads it. */
495 ph->elements_used = 0;
496 add_small_page_to_freelist(head, ph, sp_bits);
502 static u16 get_small_page(struct header *head, unsigned long poolsize,
503 unsigned int sp_bits)
507 ret = pop_from_list(head, &head->small_free_list, sp_bits);
510 ret = get_large_page(head, poolsize, sp_bits);
512 ret = break_up_large_page(head, sp_bits, ret);
516 static bool huge_allocated(struct header *head, unsigned long offset)
519 struct huge_alloc *ha;
521 for (i = head->huge; i; i = ha->next) {
522 ha = (void *)((char *)head + i);
523 if (ha->off <= offset && ha->off + ha->len > offset)
529 /* They want something really big. Aim for contiguous pages (slow). */
530 static COLD void *huge_alloc(void *pool, unsigned long poolsize,
531 unsigned long size, unsigned long align)
533 struct header *head = pool;
534 struct huge_alloc *ha;
535 unsigned long i, sp_bits, lp_bits, num, header_size;
537 sp_bits = small_page_bits(poolsize);
538 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
540 /* Allocate tracking structure optimistically. */
541 ha = alloc_get(pool, poolsize, sizeof(*ha), ALIGNOF(*ha));
545 /* First search for contiguous small pages... */
546 header_size = sizeof(*head) + sizeof(head->bs) * (max_bucket(lp_bits)-1);
549 for (i = (header_size + (1UL << sp_bits) - 1) >> sp_bits;
550 i << sp_bits < poolsize;
552 struct page_header *pg;
553 unsigned long off = (i << sp_bits);
555 /* Skip over large pages. */
556 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
557 i += (1UL << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
561 /* Does this page meet alignment requirements? */
562 if (!num && off % align != 0)
565 /* FIXME: This makes us O(n^2). */
566 if (huge_allocated(head, off)) {
571 pg = (struct page_header *)((char *)head + off);
572 if (pg->elements_used) {
578 if (num << sp_bits >= size) {
581 /* Remove from free list. */
582 for (pgnum = i; pgnum > i - num; pgnum--) {
583 pg = from_pgnum(head, pgnum, sp_bits);
585 &head->small_free_list,
588 ha->off = (i - num + 1) << sp_bits;
589 ha->len = num << sp_bits;
594 /* Now search for large pages... */
595 recombine_small_pages(head, poolsize, sp_bits);
598 for (i = (header_size + (1UL << lp_bits) - 1) >> lp_bits;
599 (i << lp_bits) < poolsize; i++) {
600 struct page_header *pg;
601 unsigned long off = (i << lp_bits);
603 /* Ignore small pages. */
604 if (!test_bit(head->pagesize, i))
607 /* Does this page meet alignment requirements? */
608 if (!num && off % align != 0)
611 /* FIXME: This makes us O(n^2). */
612 if (huge_allocated(head, off)) {
617 pg = (struct page_header *)((char *)head + off);
618 if (pg->elements_used) {
624 if (num << lp_bits >= size) {
627 /* Remove from free list. */
628 for (pgnum = i; pgnum > i - num; pgnum--) {
629 pg = from_pgnum(head, pgnum, lp_bits);
631 &head->large_free_list,
634 ha->off = (i - num + 1) << lp_bits;
635 ha->len = num << lp_bits;
640 /* Unable to satisfy: free huge alloc structure. */
641 alloc_free(pool, poolsize, ha);
645 add_to_huge_list(pool, ha);
646 return (char *)pool + ha->off;
650 huge_free(struct header *head, unsigned long poolsize, void *free)
652 unsigned long i, off, pgnum, free_off = (char *)free - (char *)head;
653 unsigned int sp_bits, lp_bits;
654 struct huge_alloc *ha;
656 for (i = head->huge; i; i = ha->next) {
657 ha = (void *)((char *)head + i);
658 if (free_off == ha->off)
663 /* Free up all the pages, delete and free ha */
664 sp_bits = small_page_bits(poolsize);
665 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
666 pgnum = free_off >> sp_bits;
668 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
670 off < ha->off + ha->len;
671 off += 1UL << lp_bits) {
672 add_large_page_to_freelist(head,
673 (void *)((char *)head + off),
678 off < ha->off + ha->len;
679 off += 1UL << sp_bits) {
680 add_small_page_to_freelist(head,
681 (void *)((char *)head + off),
685 del_from_huge(head, ha);
686 alloc_free(head, poolsize, ha);
689 static COLD unsigned long huge_size(struct header *head, void *p)
691 unsigned long i, off = (char *)p - (char *)head;
692 struct huge_alloc *ha;
694 for (i = head->huge; i; i = ha->next) {
695 ha = (void *)((char *)head + i);
696 if (off == ha->off) {
703 void *alloc_get(void *pool, unsigned long poolsize,
704 unsigned long size, unsigned long align)
706 struct header *head = pool;
709 struct bucket_state *bs;
710 struct page_header *ph;
711 unsigned int sp_bits;
713 if (poolsize < MIN_USEFUL_SIZE) {
714 return tiny_alloc_get(pool, poolsize, size, align);
717 size = align_up(size, align);
720 bucket = size_to_bucket(size);
722 sp_bits = small_page_bits(poolsize);
724 if (bucket >= max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE)) {
725 return huge_alloc(pool, poolsize, size, align);
728 bs = &head->bs[bucket];
730 if (!bs->page_list) {
731 struct page_header *ph;
733 if (large_page_bucket(bucket, sp_bits))
734 bs->page_list = get_large_page(head, poolsize,
737 bs->page_list = get_small_page(head, poolsize,
739 /* FIXME: Try large-aligned alloc? Header stuffing? */
740 if (unlikely(!bs->page_list))
742 ph = from_pgnum(head, bs->page_list, sp_bits);
744 ph->elements_used = 0;
746 memset(ph->used, 0, used_size(bs->elements_per_page));
749 ph = from_pgnum(head, bs->page_list, sp_bits);
751 i = find_free_bit(ph->used);
752 set_bit(ph->used, i);
755 /* check if this page is now full */
756 if (unlikely(ph->elements_used == bs->elements_per_page)) {
757 del_from_bucket_list(head, bs, ph, sp_bits);
758 add_to_bucket_full_list(head, bs, ph, sp_bits);
761 return (char *)ph + page_header_size(ph->bucket / INTER_BUCKET_SPACE,
762 bs->elements_per_page)
763 + i * bucket_to_size(bucket);
766 void alloc_free(void *pool, unsigned long poolsize, void *free)
768 struct header *head = pool;
769 struct bucket_state *bs;
770 unsigned int sp_bits;
771 unsigned long i, pgnum, pgoffset, offset = (char *)free - (char *)pool;
773 struct page_header *ph;
775 if (poolsize < MIN_USEFUL_SIZE) {
776 tiny_alloc_free(pool, poolsize, free);
780 /* Get page header. */
781 sp_bits = small_page_bits(poolsize);
782 pgnum = offset >> sp_bits;
784 /* Big page? Round down further. */
785 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
787 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
791 /* Step back to page header. */
792 ph = from_pgnum(head, pgnum, sp_bits);
793 if ((void *)ph == free) {
794 huge_free(head, poolsize, free);
798 bs = &head->bs[ph->bucket];
799 pgoffset = offset - (pgnum << sp_bits)
800 - page_header_size(ph->bucket / INTER_BUCKET_SPACE,
801 bs->elements_per_page);
803 if (unlikely(ph->elements_used == bs->elements_per_page)) {
804 del_from_bucket_full_list(head, bs, ph, sp_bits);
805 add_to_bucket_list(head, bs, ph, sp_bits);
808 /* Which element are we? */
809 i = pgoffset / bucket_to_size(ph->bucket);
810 clear_bit(ph->used, i);
813 if (unlikely(ph->elements_used == 0)) {
814 bs = &head->bs[ph->bucket];
815 del_from_bucket_list(head, bs, ph, sp_bits);
817 add_small_page_to_freelist(head, ph, sp_bits);
819 add_large_page_to_freelist(head, ph, sp_bits);
823 unsigned long alloc_size(void *pool, unsigned long poolsize, void *p)
825 struct header *head = pool;
826 unsigned int pgnum, sp_bits;
827 unsigned long offset = (char *)p - (char *)pool;
828 struct page_header *ph;
830 if (poolsize < MIN_USEFUL_SIZE)
831 return tiny_alloc_size(pool, poolsize, p);
833 /* Get page header. */
834 sp_bits = small_page_bits(poolsize);
835 pgnum = offset >> sp_bits;
837 /* Big page? Round down further. */
838 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE))
839 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
841 /* Step back to page header. */
842 ph = from_pgnum(head, pgnum, sp_bits);
844 return huge_size(head, p);
846 return bucket_to_size(ph->bucket);
849 /* Useful for gdb breakpoints. */
850 static bool check_fail(void)
855 static unsigned long count_bits(const unsigned long bitmap[],
858 unsigned long i, count = 0;
860 while (limit >= BITS_PER_LONG) {
861 count += popcount(bitmap[0]);
863 limit -= BITS_PER_LONG;
866 for (i = 0; i < limit; i++)
867 if (test_bit(bitmap, i))
872 static bool out_of_bounds(unsigned long pgnum,
873 unsigned int sp_bits,
874 unsigned long pagesize,
875 unsigned long poolsize)
877 if (((pgnum << sp_bits) >> sp_bits) != pgnum)
880 if ((pgnum << sp_bits) > poolsize)
883 return ((pgnum << sp_bits) + pagesize > poolsize);
886 static bool check_bucket(struct header *head,
887 unsigned long poolsize,
888 unsigned long pages[],
889 struct bucket_state *bs,
893 struct page_header *ph;
894 unsigned long taken, i, prev, pagesize, sp_bits, lp_bits;
896 sp_bits = small_page_bits(poolsize);
897 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
899 lp_bucket = large_page_bucket(bindex, sp_bits);
901 pagesize = 1UL << (lp_bucket ? lp_bits : sp_bits);
903 /* This many elements fit? */
904 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
905 bs->elements_per_page);
906 taken += bucket_to_size(bindex) * bs->elements_per_page;
907 if (taken > pagesize)
910 /* One more wouldn't fit? */
911 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
912 bs->elements_per_page + 1);
913 taken += bucket_to_size(bindex) * (bs->elements_per_page + 1);
914 if (taken <= pagesize)
917 /* Walk used list. */
919 for (i = bs->page_list; i; i = ph->next) {
921 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
923 /* Wrong size page? */
924 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
927 /* Large page not on boundary? */
928 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
930 ph = from_pgnum(head, i, sp_bits);
931 /* Linked list corrupt? */
932 if (ph->prev != prev)
934 /* Already seen this page? */
935 if (test_bit(pages, i))
939 if (ph->elements_used == 0)
941 if (ph->elements_used >= bs->elements_per_page)
943 /* Used bits don't agree? */
944 if (ph->elements_used != count_bits(ph->used,
945 bs->elements_per_page))
948 if (ph->bucket != bindex)
953 /* Walk full list. */
955 for (i = bs->full_list; i; i = ph->next) {
957 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
959 /* Wrong size page? */
960 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
962 /* Large page not on boundary? */
963 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
965 ph = from_pgnum(head, i, sp_bits);
966 /* Linked list corrupt? */
967 if (ph->prev != prev)
969 /* Already seen this page? */
970 if (test_bit(pages, i))
974 if (ph->elements_used != bs->elements_per_page)
976 /* Used bits don't agree? */
977 if (ph->elements_used != count_bits(ph->used,
978 bs->elements_per_page))
981 if (ph->bucket != bindex)
988 bool alloc_check(void *pool, unsigned long poolsize)
990 struct header *head = pool;
991 unsigned long prev, i, lp_bits, sp_bits, header_size, num_buckets;
992 struct page_header *ph;
993 struct huge_alloc *ha;
994 unsigned long pages[MAX_SMALL_PAGES / BITS_PER_LONG] = { 0 };
996 if (poolsize < MIN_USEFUL_SIZE)
997 return tiny_alloc_check(pool, poolsize);
999 sp_bits = small_page_bits(poolsize);
1000 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1002 num_buckets = max_bucket(lp_bits);
1004 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1006 /* First, set all bits taken by header. */
1007 for (i = 0; i < header_size; i += (1UL << sp_bits))
1008 set_bit(pages, i >> sp_bits);
1010 /* Check small page free list. */
1012 for (i = head->small_free_list; i; i = ph->next) {
1014 if (out_of_bounds(i, sp_bits, 1UL << sp_bits, poolsize))
1015 return check_fail();
1017 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1018 return check_fail();
1019 ph = from_pgnum(head, i, sp_bits);
1020 /* Linked list corrupt? */
1021 if (ph->prev != prev)
1022 return check_fail();
1023 /* Already seen this page? */
1024 if (test_bit(pages, i))
1025 return check_fail();
1030 /* Check large page free list. */
1032 for (i = head->large_free_list; i; i = ph->next) {
1034 if (out_of_bounds(i, sp_bits, 1UL << lp_bits, poolsize))
1035 return check_fail();
1036 /* Not large page? */
1037 if (!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1038 return check_fail();
1039 /* Not page boundary? */
1040 if ((i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
1041 return check_fail();
1042 ph = from_pgnum(head, i, sp_bits);
1043 /* Linked list corrupt? */
1044 if (ph->prev != prev)
1045 return check_fail();
1046 /* Already seen this page? */
1047 if (test_bit(pages, i))
1048 return check_fail();
1053 /* Check the buckets. */
1054 for (i = 0; i < max_bucket(lp_bits); i++) {
1055 struct bucket_state *bs = &head->bs[i];
1057 if (!check_bucket(head, poolsize, pages, bs, i))
1061 /* Check the huge alloc list. */
1063 for (i = head->huge; i; i = ha->next) {
1064 unsigned long pgbits, j;
1067 if (i >= poolsize || i + sizeof(*ha) > poolsize)
1068 return check_fail();
1069 ha = (void *)((char *)head + i);
1071 /* Check contents of ha. */
1072 if (ha->off > poolsize || ha->off + ha->len > poolsize)
1073 return check_fail();
1075 /* Large or small page? */
1076 pgbits = test_bit(head->pagesize, ha->off >> lp_bits)
1077 ? lp_bits : sp_bits;
1079 /* Not page boundary? */
1080 if ((ha->off % (1UL << pgbits)) != 0)
1081 return check_fail();
1083 /* Not page length? */
1084 if ((ha->len % (1UL << pgbits)) != 0)
1085 return check_fail();
1087 /* Linked list corrupt? */
1088 if (ha->prev != prev)
1089 return check_fail();
1091 for (j = ha->off; j < ha->off + ha->len; j += (1UL<<sp_bits)) {
1092 /* Already seen this page? */
1093 if (test_bit(pages, j >> sp_bits))
1094 return check_fail();
1095 set_bit(pages, j >> sp_bits);
1101 /* Make sure every page accounted for. */
1102 for (i = 0; i < poolsize >> sp_bits; i++) {
1103 if (!test_bit(pages, i))
1104 return check_fail();
1105 if (test_bit(head->pagesize,
1106 i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
1107 /* Large page, skip rest. */
1108 i += SMALL_PAGES_PER_LARGE_PAGE - 1;
1115 static unsigned long print_overhead(FILE *out, const char *desc,
1116 unsigned long bytes,
1117 unsigned long poolsize)
1119 fprintf(out, "Overhead (%s): %lu bytes (%.3g%%)\n",
1120 desc, bytes, 100.0 * bytes / poolsize);
1124 static unsigned long count_list(struct header *head,
1126 unsigned int sp_bits,
1127 unsigned long *total_elems)
1129 struct page_header *p;
1130 unsigned long ret = 0;
1133 p = from_pgnum(head, pgnum, sp_bits);
1135 (*total_elems) += p->elements_used;
1142 static unsigned long visualize_bucket(FILE *out, struct header *head,
1143 unsigned int bucket,
1144 unsigned long poolsize,
1145 unsigned int sp_bits)
1147 unsigned long num_full, num_partial, num_pages, page_size,
1148 elems, hdr_min, hdr_size, elems_per_page, overhead = 0;
1150 elems_per_page = head->bs[bucket].elements_per_page;
1152 /* If we used byte-based bitmaps, we could get pg hdr to: */
1153 hdr_min = sizeof(struct page_header)
1154 - sizeof(((struct page_header *)0)->used)
1155 + align_up(elems_per_page, CHAR_BIT) / CHAR_BIT;
1156 hdr_size = page_header_size(bucket / INTER_BUCKET_SPACE,
1160 num_full = count_list(head, head->bs[bucket].full_list, sp_bits,
1162 num_partial = count_list(head, head->bs[bucket].page_list, sp_bits,
1164 num_pages = num_full + num_partial;
1168 fprintf(out, "Bucket %u (%lu bytes):"
1169 " %lu full, %lu partial = %lu elements\n",
1170 bucket, bucket_to_size(bucket), num_full, num_partial, elems);
1171 /* Strict requirement of page header size. */
1172 overhead += print_overhead(out, "page headers",
1173 hdr_min * num_pages, poolsize);
1174 /* Gap between minimal page header and actual start. */
1175 overhead += print_overhead(out, "page post-header alignments",
1176 (hdr_size - hdr_min) * num_pages, poolsize);
1177 /* Between last element and end of page. */
1178 page_size = (1UL << sp_bits);
1179 if (large_page_bucket(bucket, sp_bits))
1180 page_size <<= BITS_FROM_SMALL_TO_LARGE_PAGE;
1182 overhead += print_overhead(out, "page tails",
1183 (page_size - (hdr_size
1185 * bucket_to_size(bucket))))
1186 * num_pages, poolsize);
1190 void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
1192 struct header *head = pool;
1193 unsigned long i, lp_bits, sp_bits, header_size, num_buckets, count,
1196 fprintf(out, "Pool %p size %lu: (%s allocator)\n", pool, poolsize,
1197 poolsize < MIN_USEFUL_SIZE ? "tiny" : "standard");
1199 if (poolsize < MIN_USEFUL_SIZE) {
1200 tiny_alloc_visualize(out, pool, poolsize);
1204 sp_bits = small_page_bits(poolsize);
1205 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1207 num_buckets = max_bucket(lp_bits);
1208 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1210 fprintf(out, "Large page size %lu, small page size %lu.\n",
1211 1UL << lp_bits, 1UL << sp_bits);
1212 overhead += print_overhead(out, "unused pool tail",
1213 poolsize % (1UL << lp_bits), poolsize);
1214 fprintf(out, "Main header %lu bytes (%lu small pages).\n",
1215 header_size, align_up(header_size, 1UL << sp_bits) >> sp_bits);
1216 overhead += print_overhead(out, "partial header page",
1217 align_up(header_size, 1UL << sp_bits)
1218 - header_size, poolsize);
1219 /* Total large pages. */
1220 i = count_bits(head->pagesize, poolsize >> lp_bits);
1222 count = i - count_list(head, head->large_free_list, sp_bits, NULL);
1223 fprintf(out, "%lu/%lu large pages used (%.3g%%)\n",
1224 count, i, count ? 100.0 * count / i : 0.0);
1226 /* Total small pages. */
1227 i = ((poolsize >> lp_bits) - i) << BITS_FROM_SMALL_TO_LARGE_PAGE;
1229 count = i - count_list(head, head->small_free_list, sp_bits, NULL);
1230 fprintf(out, "%lu/%lu small pages used (%.3g%%)\n",
1231 count, i, count ? 100.0 * count / i : 0.0);
1233 /* Summary of each bucket. */
1234 fprintf(out, "%lu buckets:\n", num_buckets);
1235 for (i = 0; i < num_buckets; i++)
1236 overhead += visualize_bucket(out, head, i, poolsize, sp_bits);
1238 print_overhead(out, "total", overhead, poolsize);