10 #include <ccan/build_assert/build_assert.h>
11 #include <ccan/likely/likely.h>
12 #include <ccan/alignof/alignof.h>
13 #include <ccan/short_types/short_types.h>
14 #include <ccan/compiler/compiler.h>
18 Inspired by (and parts taken from) Andrew Tridgell's alloc_mmap:
19 http://samba.org/~tridge/junkcode/alloc_mmap/
21 Copyright (C) Andrew Tridgell 2007
23 This library is free software; you can redistribute it and/or
24 modify it under the terms of the GNU Lesser General Public
25 License as published by the Free Software Foundation; either
26 version 2 of the License, or (at your option) any later version.
28 This library is distributed in the hope that it will be useful,
29 but WITHOUT ANY WARRANTY; without even the implied warranty of
30 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
31 Lesser General Public License for more details.
33 You should have received a copy of the GNU Lesser General Public
34 License along with this library; if not, write to the Free Software
35 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
38 /* We divide the pool into this many large pages (nearest power of 2) */
39 #define MAX_LARGE_PAGES (256UL)
41 /* 32 small pages == 1 large page. */
42 #define BITS_FROM_SMALL_TO_LARGE_PAGE 5
44 #define MAX_SMALL_PAGES (MAX_LARGE_PAGES << BITS_FROM_SMALL_TO_LARGE_PAGE)
46 /* Smallest pool size for this scheme: 128-byte small pages. That's
47 * 9/13% overhead for 32/64 bit. */
48 #define MIN_USEFUL_SIZE (MAX_SMALL_PAGES * 128)
50 /* Every 4 buckets, we jump up a power of 2. ...8 10 12 14 16 20 24 28 32... */
51 #define INTER_BUCKET_SPACE 4
53 #define SMALL_PAGES_PER_LARGE_PAGE (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)
55 /* FIXME: Figure this out properly. */
56 #define MAX_SIZE (1 << 30)
58 /* How few object to fit in a page before using a larger one? (8) */
59 #define MAX_PAGE_OBJECT_ORDER 3
61 #define BITS_PER_LONG (sizeof(long) * CHAR_BIT)
64 u32 elements_per_page;
70 /* Bitmap of which pages are large. */
71 unsigned long pagesize[MAX_LARGE_PAGES / BITS_PER_LONG];
73 /* List of unused small/large pages. */
77 /* List of huge allocs. */
80 /* This is less defined: we have two buckets for each power of 2 */
81 struct bucket_state bs[1];
85 unsigned long next, prev;
86 unsigned long off, len;
91 /* FIXME: We can just count all-0 and all-1 used[] elements. */
92 unsigned elements_used : 25;
94 unsigned long used[1]; /* One bit per element. */
98 * Every 4 buckets, the size doubles.
99 * Between buckets, sizes increase linearly.
101 * eg. bucket 40 = 2^10 = 1024
102 * bucket 41 = 2^10 + 2^10*4 = 1024 + 256
103 * bucket 42 = 2^10 + 2^10*4 = 1024 + 512
104 * bucket 43 = 2^10 + 2^10*4 = 1024 + 768
105 * bucket 45 = 2^11 = 2048
107 * Care is taken to handle low numbered buckets, at cost of overflow.
109 static unsigned long bucket_to_size(unsigned int bucket)
111 unsigned long base = 1UL << (bucket / INTER_BUCKET_SPACE);
112 return base + ((bucket % INTER_BUCKET_SPACE)
113 << (bucket / INTER_BUCKET_SPACE))
114 / INTER_BUCKET_SPACE;
119 * fls(size/2) == 3. 1 << 3 == 8, so we're 2 too large, out of a possible
120 * 8 too large. That's 1/4 of the way to the next power of 2 == 1 bucket.
122 * We make sure we round up. Note that this fails on 32 bit at size
123 * 1879048193 (around bucket 120).
125 static unsigned int size_to_bucket(unsigned long size)
127 unsigned int base = fls(size/2);
128 unsigned long overshoot;
130 overshoot = size - (1UL << base);
131 return base * INTER_BUCKET_SPACE
132 + ((overshoot * INTER_BUCKET_SPACE + (1UL << base)-1) >> base);
135 static unsigned int small_page_bits(unsigned long poolsize)
137 return fls(poolsize / MAX_SMALL_PAGES - 1);
140 static struct page_header *from_pgnum(struct header *head,
144 return (struct page_header *)((char *)head + (pgnum << sp_bits));
147 static u16 to_pgnum(struct header *head, void *p, unsigned sp_bits)
149 return ((char *)p - (char *)head) >> sp_bits;
152 static size_t used_size(unsigned int num_elements)
154 return align_up(num_elements, BITS_PER_LONG) / CHAR_BIT;
158 * We always align the first entry to the lower power of 2.
159 * eg. the 12-byte bucket gets 8-byte aligned. The 4096-byte bucket
160 * gets 4096-byte aligned.
162 static unsigned long page_header_size(unsigned int align_bits,
163 unsigned long num_elements)
167 size = sizeof(struct page_header)
168 - sizeof(((struct page_header *)0)->used)
169 + used_size(num_elements);
170 return align_up(size, 1UL << align_bits);
173 static void add_to_list(struct header *head,
174 u16 *list, struct page_header *ph, unsigned sp_bits)
176 unsigned long h = *list, offset = to_pgnum(head, ph, sp_bits);
180 struct page_header *prev = from_pgnum(head, h, sp_bits);
181 assert(prev->prev == 0);
188 static void del_from_list(struct header *head,
189 u16 *list, struct page_header *ph, unsigned sp_bits)
195 struct page_header *prev = from_pgnum(head, ph->prev, sp_bits);
196 prev->next = ph->next;
199 struct page_header *next = from_pgnum(head, ph->next, sp_bits);
200 next->prev = ph->prev;
204 static u16 pop_from_list(struct header *head,
206 unsigned int sp_bits)
209 struct page_header *ph = from_pgnum(head, h, sp_bits);
214 from_pgnum(head, *list, sp_bits)->prev = 0;
219 static void add_to_huge_list(struct header *head, struct huge_alloc *ha)
221 unsigned long h = head->huge;
222 unsigned long offset = (char *)ha - (char *)head;
226 struct huge_alloc *prev = (void *)((char *)head + h);
227 assert(prev->prev == 0);
234 static void del_from_huge(struct header *head, struct huge_alloc *ha)
238 head->huge = ha->next;
240 struct huge_alloc *prev = (void *)((char *)head + ha->prev);
241 prev->next = ha->next;
244 struct huge_alloc *next = (void *)((char *)head + ha->next);
245 next->prev = ha->prev;
249 static void add_small_page_to_freelist(struct header *head,
250 struct page_header *ph,
251 unsigned int sp_bits)
253 add_to_list(head, &head->small_free_list, ph, sp_bits);
256 static void add_large_page_to_freelist(struct header *head,
257 struct page_header *ph,
258 unsigned int sp_bits)
260 add_to_list(head, &head->large_free_list, ph, sp_bits);
263 static void add_to_bucket_list(struct header *head,
264 struct bucket_state *bs,
265 struct page_header *ph,
266 unsigned int sp_bits)
268 add_to_list(head, &bs->page_list, ph, sp_bits);
271 static void del_from_bucket_list(struct header *head,
272 struct bucket_state *bs,
273 struct page_header *ph,
274 unsigned int sp_bits)
276 del_from_list(head, &bs->page_list, ph, sp_bits);
279 static void del_from_bucket_full_list(struct header *head,
280 struct bucket_state *bs,
281 struct page_header *ph,
282 unsigned int sp_bits)
284 del_from_list(head, &bs->full_list, ph, sp_bits);
287 static void add_to_bucket_full_list(struct header *head,
288 struct bucket_state *bs,
289 struct page_header *ph,
290 unsigned int sp_bits)
292 add_to_list(head, &bs->full_list, ph, sp_bits);
295 static void clear_bit(unsigned long bitmap[], unsigned int off)
297 bitmap[off / BITS_PER_LONG] &= ~(1UL << (off % BITS_PER_LONG));
300 static bool test_bit(const unsigned long bitmap[], unsigned int off)
302 return bitmap[off / BITS_PER_LONG] & (1UL << (off % BITS_PER_LONG));
305 static void set_bit(unsigned long bitmap[], unsigned int off)
307 bitmap[off / BITS_PER_LONG] |= (1UL << (off % BITS_PER_LONG));
310 /* There must be a bit to be found. */
311 static unsigned int find_free_bit(const unsigned long bitmap[])
315 for (i = 0; bitmap[i] == -1UL; i++);
316 return (i*BITS_PER_LONG) + ffsl(~bitmap[i]) - 1;
319 /* How many elements can we fit in a page? */
320 static unsigned long elements_per_page(unsigned long align_bits,
324 unsigned long num, overhead;
326 /* First approximation: no extra room for bitmap. */
327 overhead = align_up(sizeof(struct page_header), 1UL << align_bits);
328 num = (psize - overhead) / esize;
330 while (page_header_size(align_bits, num) + esize * num > psize)
335 static bool large_page_bucket(unsigned int bucket, unsigned int sp_bits)
337 unsigned long max_smallsize;
339 /* Note: this doesn't take into account page header. */
340 max_smallsize = (1UL << sp_bits) >> MAX_PAGE_OBJECT_ORDER;
342 return bucket_to_size(bucket) > max_smallsize;
345 static unsigned int max_bucket(unsigned int lp_bits)
347 return (lp_bits - MAX_PAGE_OBJECT_ORDER) * INTER_BUCKET_SPACE;
350 void alloc_init(void *pool, unsigned long poolsize)
352 struct header *head = pool;
353 struct page_header *ph;
354 unsigned int lp_bits, sp_bits, num_buckets;
355 unsigned long header_size, i;
357 if (poolsize < MIN_USEFUL_SIZE) {
358 tiny_alloc_init(pool, poolsize);
362 /* We rely on page numbers fitting in 16 bit. */
363 BUILD_ASSERT(MAX_SMALL_PAGES < 65536);
365 sp_bits = small_page_bits(poolsize);
366 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
368 num_buckets = max_bucket(lp_bits);
371 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
373 memset(head, 0, header_size);
374 for (i = 0; i < num_buckets; i++) {
375 unsigned long pagesize;
377 if (large_page_bucket(i, sp_bits))
378 pagesize = 1UL << lp_bits;
380 pagesize = 1UL << sp_bits;
382 head->bs[i].elements_per_page
383 = elements_per_page(i / INTER_BUCKET_SPACE,
388 /* They start as all large pages. */
389 memset(head->pagesize, 0xFF, sizeof(head->pagesize));
390 /* FIXME: small pages for last bit? */
392 /* Split first page into small pages. */
393 assert(header_size < (1UL << lp_bits));
394 clear_bit(head->pagesize, 0);
396 /* Skip over page(s) used by header, add rest to free list */
397 for (i = align_up(header_size, (1UL << sp_bits)) >> sp_bits;
398 i < SMALL_PAGES_PER_LARGE_PAGE;
400 ph = from_pgnum(head, i, sp_bits);
401 ph->elements_used = 0;
402 add_small_page_to_freelist(head, ph, sp_bits);
405 /* Add the rest of the pages as large pages. */
406 i = SMALL_PAGES_PER_LARGE_PAGE;
407 while ((i << sp_bits) + (1UL << lp_bits) <= poolsize) {
408 assert(i < MAX_SMALL_PAGES);
409 ph = from_pgnum(head, i, sp_bits);
410 ph->elements_used = 0;
411 add_large_page_to_freelist(head, ph, sp_bits);
412 i += SMALL_PAGES_PER_LARGE_PAGE;
416 /* A large page worth of small pages are free: delete them from free list. */
417 static void del_large_from_small_free_list(struct header *head,
418 struct page_header *ph,
419 unsigned int sp_bits)
423 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
424 del_from_list(head, &head->small_free_list,
425 (void *)ph + (i << sp_bits),
430 static bool all_empty(struct header *head,
436 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
437 struct page_header *ph = from_pgnum(head, pgnum + i, sp_bits);
438 if (ph->elements_used)
444 static void recombine_small_pages(struct header *head, unsigned long poolsize,
445 unsigned int sp_bits)
448 unsigned int lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
450 /* Look for small pages to coalesce, after first large page. */
451 for (i = SMALL_PAGES_PER_LARGE_PAGE;
452 i < (poolsize >> lp_bits) << BITS_FROM_SMALL_TO_LARGE_PAGE;
453 i += SMALL_PAGES_PER_LARGE_PAGE) {
454 /* Already a large page? */
455 if (test_bit(head->pagesize, i / SMALL_PAGES_PER_LARGE_PAGE))
457 if (all_empty(head, i, sp_bits)) {
458 struct page_header *ph = from_pgnum(head, i, sp_bits);
459 set_bit(head->pagesize,
460 i / SMALL_PAGES_PER_LARGE_PAGE);
461 del_large_from_small_free_list(head, ph, sp_bits);
462 add_large_page_to_freelist(head, ph, sp_bits);
467 static u16 get_large_page(struct header *head, unsigned long poolsize,
468 unsigned int sp_bits)
470 unsigned int lp_bits, page;
472 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
474 page = pop_from_list(head, &head->large_free_list, sp_bits);
478 recombine_small_pages(head, poolsize, sp_bits);
480 return pop_from_list(head, &head->large_free_list, sp_bits);
483 /* Returns small page. */
484 static unsigned long break_up_large_page(struct header *head,
485 unsigned int sp_bits,
490 clear_bit(head->pagesize, lpage >> BITS_FROM_SMALL_TO_LARGE_PAGE);
492 for (i = 1; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
493 struct page_header *ph = from_pgnum(head, lpage + i, sp_bits);
494 /* Initialize this: huge_alloc reads it. */
495 ph->elements_used = 0;
496 add_small_page_to_freelist(head, ph, sp_bits);
502 static u16 get_small_page(struct header *head, unsigned long poolsize,
503 unsigned int sp_bits)
507 ret = pop_from_list(head, &head->small_free_list, sp_bits);
510 ret = get_large_page(head, poolsize, sp_bits);
512 ret = break_up_large_page(head, sp_bits, ret);
516 static bool huge_allocated(struct header *head, unsigned long offset)
519 struct huge_alloc *ha;
521 for (i = head->huge; i; i = ha->next) {
522 ha = (void *)((char *)head + i);
523 if (ha->off <= offset && ha->off + ha->len > offset)
529 /* They want something really big. Aim for contiguous pages (slow). */
530 static COLD_ATTRIBUTE
531 void *huge_alloc(void *pool, unsigned long poolsize,
532 unsigned long size, unsigned long align)
534 struct header *head = pool;
535 struct huge_alloc *ha;
536 unsigned long i, sp_bits, lp_bits, num, header_size;
538 sp_bits = small_page_bits(poolsize);
539 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
541 /* Allocate tracking structure optimistically. */
542 ha = alloc_get(pool, poolsize, sizeof(*ha), ALIGNOF(*ha));
546 /* First search for contiguous small pages... */
547 header_size = sizeof(*head) + sizeof(head->bs) * (max_bucket(lp_bits)-1);
550 for (i = (header_size + (1UL << sp_bits) - 1) >> sp_bits;
551 i << sp_bits < poolsize;
553 struct page_header *pg;
554 unsigned long off = (i << sp_bits);
556 /* Skip over large pages. */
557 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
558 i += (1UL << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
562 /* Does this page meet alignment requirements? */
563 if (!num && off % align != 0)
566 /* FIXME: This makes us O(n^2). */
567 if (huge_allocated(head, off)) {
572 pg = (struct page_header *)((char *)head + off);
573 if (pg->elements_used) {
579 if (num << sp_bits >= size) {
582 /* Remove from free list. */
583 for (pgnum = i; pgnum > i - num; pgnum--) {
584 pg = from_pgnum(head, pgnum, sp_bits);
586 &head->small_free_list,
589 ha->off = (i - num + 1) << sp_bits;
590 ha->len = num << sp_bits;
595 /* Now search for large pages... */
596 recombine_small_pages(head, poolsize, sp_bits);
599 for (i = (header_size + (1UL << lp_bits) - 1) >> lp_bits;
600 (i << lp_bits) < poolsize; i++) {
601 struct page_header *pg;
602 unsigned long off = (i << lp_bits);
604 /* Ignore small pages. */
605 if (!test_bit(head->pagesize, i))
608 /* Does this page meet alignment requirements? */
609 if (!num && off % align != 0)
612 /* FIXME: This makes us O(n^2). */
613 if (huge_allocated(head, off)) {
618 pg = (struct page_header *)((char *)head + off);
619 if (pg->elements_used) {
625 if (num << lp_bits >= size) {
628 /* Remove from free list. */
629 for (pgnum = i; pgnum > i - num; pgnum--) {
630 pg = from_pgnum(head, pgnum, lp_bits);
632 &head->large_free_list,
635 ha->off = (i - num + 1) << lp_bits;
636 ha->len = num << lp_bits;
641 /* Unable to satisfy: free huge alloc structure. */
642 alloc_free(pool, poolsize, ha);
646 add_to_huge_list(pool, ha);
647 return (char *)pool + ha->off;
650 static COLD_ATTRIBUTE void
651 huge_free(struct header *head, unsigned long poolsize, void *free)
653 unsigned long i, off, pgnum, free_off = (char *)free - (char *)head;
654 unsigned int sp_bits, lp_bits;
655 struct huge_alloc *ha;
657 for (i = head->huge; i; i = ha->next) {
658 ha = (void *)((char *)head + i);
659 if (free_off == ha->off)
664 /* Free up all the pages, delete and free ha */
665 sp_bits = small_page_bits(poolsize);
666 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
667 pgnum = free_off >> sp_bits;
669 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
671 off < ha->off + ha->len;
672 off += 1UL << lp_bits) {
673 add_large_page_to_freelist(head,
674 (void *)((char *)head + off),
679 off < ha->off + ha->len;
680 off += 1UL << sp_bits) {
681 add_small_page_to_freelist(head,
682 (void *)((char *)head + off),
686 del_from_huge(head, ha);
687 alloc_free(head, poolsize, ha);
690 static COLD_ATTRIBUTE unsigned long
691 huge_size(struct header *head, void *p)
693 unsigned long i, off = (char *)p - (char *)head;
694 struct huge_alloc *ha;
696 for (i = head->huge; i; i = ha->next) {
697 ha = (void *)((char *)head + i);
698 if (off == ha->off) {
705 void *alloc_get(void *pool, unsigned long poolsize,
706 unsigned long size, unsigned long align)
708 struct header *head = pool;
711 struct bucket_state *bs;
712 struct page_header *ph;
713 unsigned int sp_bits;
715 if (poolsize < MIN_USEFUL_SIZE) {
716 return tiny_alloc_get(pool, poolsize, size, align);
719 size = align_up(size, align);
722 bucket = size_to_bucket(size);
724 sp_bits = small_page_bits(poolsize);
726 if (bucket >= max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE)) {
727 return huge_alloc(pool, poolsize, size, align);
730 bs = &head->bs[bucket];
732 if (!bs->page_list) {
733 struct page_header *ph;
735 if (large_page_bucket(bucket, sp_bits))
736 bs->page_list = get_large_page(head, poolsize,
739 bs->page_list = get_small_page(head, poolsize,
741 /* FIXME: Try large-aligned alloc? Header stuffing? */
742 if (unlikely(!bs->page_list))
744 ph = from_pgnum(head, bs->page_list, sp_bits);
746 ph->elements_used = 0;
748 memset(ph->used, 0, used_size(bs->elements_per_page));
751 ph = from_pgnum(head, bs->page_list, sp_bits);
753 i = find_free_bit(ph->used);
754 set_bit(ph->used, i);
757 /* check if this page is now full */
758 if (unlikely(ph->elements_used == bs->elements_per_page)) {
759 del_from_bucket_list(head, bs, ph, sp_bits);
760 add_to_bucket_full_list(head, bs, ph, sp_bits);
763 return (char *)ph + page_header_size(ph->bucket / INTER_BUCKET_SPACE,
764 bs->elements_per_page)
765 + i * bucket_to_size(bucket);
768 void alloc_free(void *pool, unsigned long poolsize, void *free)
770 struct header *head = pool;
771 struct bucket_state *bs;
772 unsigned int sp_bits;
773 unsigned long i, pgnum, pgoffset, offset = (char *)free - (char *)pool;
775 struct page_header *ph;
777 if (poolsize < MIN_USEFUL_SIZE) {
778 return tiny_alloc_free(pool, poolsize, free);
781 /* Get page header. */
782 sp_bits = small_page_bits(poolsize);
783 pgnum = offset >> sp_bits;
785 /* Big page? Round down further. */
786 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
788 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
792 /* Step back to page header. */
793 ph = from_pgnum(head, pgnum, sp_bits);
794 if ((void *)ph == free) {
795 huge_free(head, poolsize, free);
799 bs = &head->bs[ph->bucket];
800 pgoffset = offset - (pgnum << sp_bits)
801 - page_header_size(ph->bucket / INTER_BUCKET_SPACE,
802 bs->elements_per_page);
804 if (unlikely(ph->elements_used == bs->elements_per_page)) {
805 del_from_bucket_full_list(head, bs, ph, sp_bits);
806 add_to_bucket_list(head, bs, ph, sp_bits);
809 /* Which element are we? */
810 i = pgoffset / bucket_to_size(ph->bucket);
811 clear_bit(ph->used, i);
814 if (unlikely(ph->elements_used == 0)) {
815 bs = &head->bs[ph->bucket];
816 del_from_bucket_list(head, bs, ph, sp_bits);
818 add_small_page_to_freelist(head, ph, sp_bits);
820 add_large_page_to_freelist(head, ph, sp_bits);
824 unsigned long alloc_size(void *pool, unsigned long poolsize, void *p)
826 struct header *head = pool;
827 unsigned int pgnum, sp_bits;
828 unsigned long offset = (char *)p - (char *)pool;
829 struct page_header *ph;
831 if (poolsize < MIN_USEFUL_SIZE)
832 return tiny_alloc_size(pool, poolsize, p);
834 /* Get page header. */
835 sp_bits = small_page_bits(poolsize);
836 pgnum = offset >> sp_bits;
838 /* Big page? Round down further. */
839 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE))
840 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
842 /* Step back to page header. */
843 ph = from_pgnum(head, pgnum, sp_bits);
845 return huge_size(head, p);
847 return bucket_to_size(ph->bucket);
850 /* Useful for gdb breakpoints. */
851 static bool check_fail(void)
856 static unsigned long count_bits(const unsigned long bitmap[],
859 unsigned long i, count = 0;
861 while (limit >= BITS_PER_LONG) {
862 count += popcount(bitmap[0]);
864 limit -= BITS_PER_LONG;
867 for (i = 0; i < limit; i++)
868 if (test_bit(bitmap, i))
873 static bool out_of_bounds(unsigned long pgnum,
874 unsigned int sp_bits,
875 unsigned long pagesize,
876 unsigned long poolsize)
878 if (((pgnum << sp_bits) >> sp_bits) != pgnum)
881 if ((pgnum << sp_bits) > poolsize)
884 return ((pgnum << sp_bits) + pagesize > poolsize);
887 static bool check_bucket(struct header *head,
888 unsigned long poolsize,
889 unsigned long pages[],
890 struct bucket_state *bs,
894 struct page_header *ph;
895 unsigned long taken, i, prev, pagesize, sp_bits, lp_bits;
897 sp_bits = small_page_bits(poolsize);
898 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
900 lp_bucket = large_page_bucket(bindex, sp_bits);
902 pagesize = 1UL << (lp_bucket ? lp_bits : sp_bits);
904 /* This many elements fit? */
905 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
906 bs->elements_per_page);
907 taken += bucket_to_size(bindex) * bs->elements_per_page;
908 if (taken > pagesize)
911 /* One more wouldn't fit? */
912 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
913 bs->elements_per_page + 1);
914 taken += bucket_to_size(bindex) * (bs->elements_per_page + 1);
915 if (taken <= pagesize)
918 /* Walk used list. */
920 for (i = bs->page_list; i; i = ph->next) {
922 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
924 /* Wrong size page? */
925 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
928 /* Large page not on boundary? */
929 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
931 ph = from_pgnum(head, i, sp_bits);
932 /* Linked list corrupt? */
933 if (ph->prev != prev)
935 /* Already seen this page? */
936 if (test_bit(pages, i))
940 if (ph->elements_used == 0)
942 if (ph->elements_used >= bs->elements_per_page)
944 /* Used bits don't agree? */
945 if (ph->elements_used != count_bits(ph->used,
946 bs->elements_per_page))
949 if (ph->bucket != bindex)
954 /* Walk full list. */
956 for (i = bs->full_list; i; i = ph->next) {
958 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
960 /* Wrong size page? */
961 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
963 /* Large page not on boundary? */
964 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
966 ph = from_pgnum(head, i, sp_bits);
967 /* Linked list corrupt? */
968 if (ph->prev != prev)
970 /* Already seen this page? */
971 if (test_bit(pages, i))
975 if (ph->elements_used != bs->elements_per_page)
977 /* Used bits don't agree? */
978 if (ph->elements_used != count_bits(ph->used,
979 bs->elements_per_page))
982 if (ph->bucket != bindex)
989 bool alloc_check(void *pool, unsigned long poolsize)
991 struct header *head = pool;
992 unsigned long prev, i, lp_bits, sp_bits, header_size, num_buckets;
993 struct page_header *ph;
994 struct huge_alloc *ha;
995 unsigned long pages[MAX_SMALL_PAGES / BITS_PER_LONG] = { 0 };
997 if (poolsize < MIN_USEFUL_SIZE)
998 return tiny_alloc_check(pool, poolsize);
1000 sp_bits = small_page_bits(poolsize);
1001 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1003 num_buckets = max_bucket(lp_bits);
1005 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1007 /* First, set all bits taken by header. */
1008 for (i = 0; i < header_size; i += (1UL << sp_bits))
1009 set_bit(pages, i >> sp_bits);
1011 /* Check small page free list. */
1013 for (i = head->small_free_list; i; i = ph->next) {
1015 if (out_of_bounds(i, sp_bits, 1UL << sp_bits, poolsize))
1016 return check_fail();
1018 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1019 return check_fail();
1020 ph = from_pgnum(head, i, sp_bits);
1021 /* Linked list corrupt? */
1022 if (ph->prev != prev)
1023 return check_fail();
1024 /* Already seen this page? */
1025 if (test_bit(pages, i))
1026 return check_fail();
1031 /* Check large page free list. */
1033 for (i = head->large_free_list; i; i = ph->next) {
1035 if (out_of_bounds(i, sp_bits, 1UL << lp_bits, poolsize))
1036 return check_fail();
1037 /* Not large page? */
1038 if (!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1039 return check_fail();
1040 /* Not page boundary? */
1041 if ((i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
1042 return check_fail();
1043 ph = from_pgnum(head, i, sp_bits);
1044 /* Linked list corrupt? */
1045 if (ph->prev != prev)
1046 return check_fail();
1047 /* Already seen this page? */
1048 if (test_bit(pages, i))
1049 return check_fail();
1054 /* Check the buckets. */
1055 for (i = 0; i < max_bucket(lp_bits); i++) {
1056 struct bucket_state *bs = &head->bs[i];
1058 if (!check_bucket(head, poolsize, pages, bs, i))
1062 /* Check the huge alloc list. */
1064 for (i = head->huge; i; i = ha->next) {
1065 unsigned long pgbits, j;
1068 if (i >= poolsize || i + sizeof(*ha) > poolsize)
1069 return check_fail();
1070 ha = (void *)((char *)head + i);
1072 /* Check contents of ha. */
1073 if (ha->off > poolsize || ha->off + ha->len > poolsize)
1074 return check_fail();
1076 /* Large or small page? */
1077 pgbits = test_bit(head->pagesize, ha->off >> lp_bits)
1078 ? lp_bits : sp_bits;
1080 /* Not page boundary? */
1081 if ((ha->off % (1UL << pgbits)) != 0)
1082 return check_fail();
1084 /* Not page length? */
1085 if ((ha->len % (1UL << pgbits)) != 0)
1086 return check_fail();
1088 /* Linked list corrupt? */
1089 if (ha->prev != prev)
1090 return check_fail();
1092 for (j = ha->off; j < ha->off + ha->len; j += (1UL<<sp_bits)) {
1093 /* Already seen this page? */
1094 if (test_bit(pages, j >> sp_bits))
1095 return check_fail();
1096 set_bit(pages, j >> sp_bits);
1102 /* Make sure every page accounted for. */
1103 for (i = 0; i < poolsize >> sp_bits; i++) {
1104 if (!test_bit(pages, i))
1105 return check_fail();
1106 if (test_bit(head->pagesize,
1107 i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
1108 /* Large page, skip rest. */
1109 i += SMALL_PAGES_PER_LARGE_PAGE - 1;
1116 static unsigned long print_overhead(FILE *out, const char *desc,
1117 unsigned long bytes,
1118 unsigned long poolsize)
1120 fprintf(out, "Overhead (%s): %lu bytes (%.3g%%)\n",
1121 desc, bytes, 100.0 * bytes / poolsize);
1125 static unsigned long count_list(struct header *head,
1127 unsigned int sp_bits,
1128 unsigned long *total_elems)
1130 struct page_header *p;
1131 unsigned long ret = 0;
1134 p = from_pgnum(head, pgnum, sp_bits);
1136 (*total_elems) += p->elements_used;
1143 static unsigned long visualize_bucket(FILE *out, struct header *head,
1144 unsigned int bucket,
1145 unsigned long poolsize,
1146 unsigned int sp_bits)
1148 unsigned long num_full, num_partial, num_pages, page_size,
1149 elems, hdr_min, hdr_size, elems_per_page, overhead = 0;
1151 elems_per_page = head->bs[bucket].elements_per_page;
1153 /* If we used byte-based bitmaps, we could get pg hdr to: */
1154 hdr_min = sizeof(struct page_header)
1155 - sizeof(((struct page_header *)0)->used)
1156 + align_up(elems_per_page, CHAR_BIT) / CHAR_BIT;
1157 hdr_size = page_header_size(bucket / INTER_BUCKET_SPACE,
1161 num_full = count_list(head, head->bs[bucket].full_list, sp_bits,
1163 num_partial = count_list(head, head->bs[bucket].page_list, sp_bits,
1165 num_pages = num_full + num_partial;
1169 fprintf(out, "Bucket %u (%lu bytes):"
1170 " %lu full, %lu partial = %lu elements\n",
1171 bucket, bucket_to_size(bucket), num_full, num_partial, elems);
1172 /* Strict requirement of page header size. */
1173 overhead += print_overhead(out, "page headers",
1174 hdr_min * num_pages, poolsize);
1175 /* Gap between minimal page header and actual start. */
1176 overhead += print_overhead(out, "page post-header alignments",
1177 (hdr_size - hdr_min) * num_pages, poolsize);
1178 /* Between last element and end of page. */
1179 page_size = (1UL << sp_bits);
1180 if (large_page_bucket(bucket, sp_bits))
1181 page_size <<= BITS_FROM_SMALL_TO_LARGE_PAGE;
1183 overhead += print_overhead(out, "page tails",
1184 (page_size - (hdr_size
1186 * bucket_to_size(bucket))))
1187 * num_pages, poolsize);
1191 void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
1193 struct header *head = pool;
1194 unsigned long i, lp_bits, sp_bits, header_size, num_buckets, count,
1197 fprintf(out, "Pool %p size %lu: (%s allocator)\n", pool, poolsize,
1198 poolsize < MIN_USEFUL_SIZE ? "tiny" : "standard");
1200 if (poolsize < MIN_USEFUL_SIZE) {
1201 tiny_alloc_visualize(out, pool, poolsize);
1205 sp_bits = small_page_bits(poolsize);
1206 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1208 num_buckets = max_bucket(lp_bits);
1209 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1211 fprintf(out, "Large page size %lu, small page size %lu.\n",
1212 1UL << lp_bits, 1UL << sp_bits);
1213 overhead += print_overhead(out, "unused pool tail",
1214 poolsize % (1UL << lp_bits), poolsize);
1215 fprintf(out, "Main header %lu bytes (%lu small pages).\n",
1216 header_size, align_up(header_size, 1UL << sp_bits) >> sp_bits);
1217 overhead += print_overhead(out, "partial header page",
1218 align_up(header_size, 1UL << sp_bits)
1219 - header_size, poolsize);
1220 /* Total large pages. */
1221 i = count_bits(head->pagesize, poolsize >> lp_bits);
1223 count = i - count_list(head, head->large_free_list, sp_bits, NULL);
1224 fprintf(out, "%lu/%lu large pages used (%.3g%%)\n",
1225 count, i, count ? 100.0 * count / i : 0.0);
1227 /* Total small pages. */
1228 i = ((poolsize >> lp_bits) - i) << BITS_FROM_SMALL_TO_LARGE_PAGE;
1230 count = i - count_list(head, head->small_free_list, sp_bits, NULL);
1231 fprintf(out, "%lu/%lu small pages used (%.3g%%)\n",
1232 count, i, count ? 100.0 * count / i : 0.0);
1234 /* Summary of each bucket. */
1235 fprintf(out, "%lu buckets:\n", num_buckets);
1236 for (i = 0; i < num_buckets; i++)
1237 overhead += visualize_bucket(out, head, i, poolsize, sp_bits);
1239 print_overhead(out, "total", overhead, poolsize);