10 #include <ccan/build_assert/build_assert.h>
11 #include <ccan/likely/likely.h>
12 #include <ccan/alignof/alignof.h>
13 #include <ccan/short_types/short_types.h>
17 Inspired by (and parts taken from) Andrew Tridgell's alloc_mmap:
18 http://samba.org/~tridge/junkcode/alloc_mmap/
20 Copyright (C) Andrew Tridgell 2007
22 This library is free software; you can redistribute it and/or
23 modify it under the terms of the GNU Lesser General Public
24 License as published by the Free Software Foundation; either
25 version 2 of the License, or (at your option) any later version.
27 This library is distributed in the hope that it will be useful,
28 but WITHOUT ANY WARRANTY; without even the implied warranty of
29 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
30 Lesser General Public License for more details.
32 You should have received a copy of the GNU Lesser General Public
33 License along with this library; if not, write to the Free Software
34 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
37 /* We divide the pool into this many large pages (nearest power of 2) */
38 #define MAX_LARGE_PAGES (1024UL)
40 /* 32 small pages == 1 large page. */
41 #define BITS_FROM_SMALL_TO_LARGE_PAGE 5
43 #define MAX_SMALL_PAGES (MAX_LARGE_PAGES << BITS_FROM_SMALL_TO_LARGE_PAGE)
45 /* Smallest pool size for this scheme: 128-byte small pages. That's
46 * 9/13% overhead for 32/64 bit. */
47 #define MIN_USEFUL_SIZE (MAX_SMALL_PAGES * 128)
49 /* Every 4 buckets, we jump up a power of 2. ...8 10 12 14 16 20 24 28 32... */
50 #define INTER_BUCKET_SPACE 4
52 #define SMALL_PAGES_PER_LARGE_PAGE (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)
54 /* FIXME: Figure this out properly. */
55 #define MAX_SIZE (1 << 30)
57 /* How few object to fit in a page before using a larger one? (8) */
58 #define MAX_PAGE_OBJECT_ORDER 3
60 #define BITS_PER_LONG (sizeof(long) * CHAR_BIT)
63 u32 elements_per_page;
69 /* Bitmap of which pages are large. */
70 unsigned long pagesize[MAX_LARGE_PAGES / BITS_PER_LONG];
72 /* List of unused small/large pages. */
76 /* List of huge allocs. */
79 /* This is less defined: we have two buckets for each power of 2 */
80 struct bucket_state bs[1];
84 unsigned long next, prev;
85 unsigned long off, len;
90 /* FIXME: We can just count all-0 and all-1 used[] elements. */
91 unsigned elements_used : 25;
93 unsigned long used[1]; /* One bit per element. */
97 * Every 4 buckets, the size doubles.
98 * Between buckets, sizes increase linearly.
100 * eg. bucket 40 = 2^10 = 1024
101 * bucket 41 = 2^10 + 2^10*4 = 1024 + 256
102 * bucket 42 = 2^10 + 2^10*4 = 1024 + 512
103 * bucket 43 = 2^10 + 2^10*4 = 1024 + 768
104 * bucket 45 = 2^11 = 2048
106 * Care is taken to handle low numbered buckets, at cost of overflow.
108 static unsigned long bucket_to_size(unsigned int bucket)
110 unsigned long base = 1 << (bucket / INTER_BUCKET_SPACE);
111 return base + ((bucket % INTER_BUCKET_SPACE)
112 << (bucket / INTER_BUCKET_SPACE))
113 / INTER_BUCKET_SPACE;
118 * fls(size/2) == 3. 1 << 3 == 8, so we're 2 too large, out of a possible
119 * 8 too large. That's 1/4 of the way to the next power of 2 == 1 bucket.
121 * We make sure we round up. Note that this fails on 32 bit at size
122 * 1879048193 (around bucket 120).
124 static unsigned int size_to_bucket(unsigned long size)
126 unsigned int base = fls(size/2);
127 unsigned long overshoot;
129 overshoot = size - (1 << base);
130 return base * INTER_BUCKET_SPACE
131 + ((overshoot * INTER_BUCKET_SPACE + (1 << base)-1) >> base);
134 static unsigned int small_page_bits(unsigned long poolsize)
136 return fls(poolsize / MAX_SMALL_PAGES / 2);
139 static struct page_header *from_pgnum(struct header *head,
143 return (struct page_header *)((char *)head + (pgnum << sp_bits));
146 static u16 to_pgnum(struct header *head, void *p, unsigned sp_bits)
148 return ((char *)p - (char *)head) >> sp_bits;
151 static size_t used_size(unsigned int num_elements)
153 return align_up(num_elements, BITS_PER_LONG) / CHAR_BIT;
157 * We always align the first entry to the lower power of 2.
158 * eg. the 12-byte bucket gets 8-byte aligned. The 4096-byte bucket
159 * gets 4096-byte aligned.
161 static unsigned long page_header_size(unsigned int align_bits,
162 unsigned long num_elements)
166 size = sizeof(struct page_header)
167 - sizeof(((struct page_header *)0)->used)
168 + used_size(num_elements);
169 return align_up(size, 1 << align_bits);
172 static void add_to_list(struct header *head,
173 u16 *list, struct page_header *ph, unsigned sp_bits)
175 unsigned long h = *list, offset = to_pgnum(head, ph, sp_bits);
179 struct page_header *prev = from_pgnum(head, h, sp_bits);
180 assert(prev->prev == 0);
187 static void del_from_list(struct header *head,
188 u16 *list, struct page_header *ph, unsigned sp_bits)
194 struct page_header *prev = from_pgnum(head, ph->prev, sp_bits);
195 prev->next = ph->next;
198 struct page_header *next = from_pgnum(head, ph->next, sp_bits);
199 next->prev = ph->prev;
203 static u16 pop_from_list(struct header *head,
205 unsigned int sp_bits)
208 struct page_header *ph = from_pgnum(head, h, sp_bits);
213 from_pgnum(head, *list, sp_bits)->prev = 0;
218 static void add_to_huge_list(struct header *head, struct huge_alloc *ha)
220 unsigned long h = head->huge;
221 unsigned long offset = (char *)ha - (char *)head;
225 struct huge_alloc *prev = (void *)((char *)head + h);
226 assert(prev->prev == 0);
233 static void del_from_huge(struct header *head, struct huge_alloc *ha)
237 head->huge = ha->next;
239 struct huge_alloc *prev = (void *)((char *)head + ha->prev);
240 prev->next = ha->next;
243 struct huge_alloc *next = (void *)((char *)head + ha->next);
244 next->prev = ha->prev;
248 static void add_small_page_to_freelist(struct header *head,
249 struct page_header *ph,
250 unsigned int sp_bits)
252 add_to_list(head, &head->small_free_list, ph, sp_bits);
255 static void add_large_page_to_freelist(struct header *head,
256 struct page_header *ph,
257 unsigned int sp_bits)
259 add_to_list(head, &head->large_free_list, ph, sp_bits);
262 static void add_to_bucket_list(struct header *head,
263 struct bucket_state *bs,
264 struct page_header *ph,
265 unsigned int sp_bits)
267 add_to_list(head, &bs->page_list, ph, sp_bits);
270 static void del_from_bucket_list(struct header *head,
271 struct bucket_state *bs,
272 struct page_header *ph,
273 unsigned int sp_bits)
275 del_from_list(head, &bs->page_list, ph, sp_bits);
278 static void del_from_bucket_full_list(struct header *head,
279 struct bucket_state *bs,
280 struct page_header *ph,
281 unsigned int sp_bits)
283 del_from_list(head, &bs->full_list, ph, sp_bits);
286 static void add_to_bucket_full_list(struct header *head,
287 struct bucket_state *bs,
288 struct page_header *ph,
289 unsigned int sp_bits)
291 add_to_list(head, &bs->full_list, ph, sp_bits);
294 static void clear_bit(unsigned long bitmap[], unsigned int off)
296 bitmap[off / BITS_PER_LONG] &= ~(1 << (off % BITS_PER_LONG));
299 static bool test_bit(const unsigned long bitmap[], unsigned int off)
301 return bitmap[off / BITS_PER_LONG] & (1 << (off % BITS_PER_LONG));
304 static void set_bit(unsigned long bitmap[], unsigned int off)
306 bitmap[off / BITS_PER_LONG] |= (1 << (off % BITS_PER_LONG));
309 /* There must be a bit to be found. */
310 static unsigned int find_free_bit(const unsigned long bitmap[])
314 for (i = 0; bitmap[i] == -1UL; i++);
315 return (i*BITS_PER_LONG) + ffsl(~bitmap[i]) - 1;
318 /* How many elements can we fit in a page? */
319 static unsigned long elements_per_page(unsigned long align_bits,
323 unsigned long num, overhead;
325 /* First approximation: no extra room for bitmap. */
326 overhead = align_up(sizeof(struct page_header), 1 << align_bits);
327 num = (psize - overhead) / esize;
329 while (page_header_size(align_bits, num) + esize * num > psize)
334 static bool large_page_bucket(unsigned int bucket, unsigned int sp_bits)
336 unsigned long max_smallsize;
338 /* Note: this doesn't take into account page header. */
339 max_smallsize = (1UL << sp_bits) >> MAX_PAGE_OBJECT_ORDER;
341 return bucket_to_size(bucket) > max_smallsize;
344 static unsigned int max_bucket(unsigned int lp_bits)
346 return (lp_bits - MAX_PAGE_OBJECT_ORDER) * INTER_BUCKET_SPACE;
349 void alloc_init(void *pool, unsigned long poolsize)
351 struct header *head = pool;
352 struct page_header *ph;
353 unsigned int lp_bits, sp_bits, num_buckets;
354 unsigned long header_size, i;
356 if (poolsize < MIN_USEFUL_SIZE) {
357 tiny_alloc_init(pool, poolsize);
361 /* We rely on page numbers fitting in 16 bit. */
362 BUILD_ASSERT(MAX_SMALL_PAGES < 65536);
364 sp_bits = small_page_bits(poolsize);
365 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
367 num_buckets = max_bucket(lp_bits);
370 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
372 memset(head, 0, header_size);
373 for (i = 0; i < num_buckets; i++) {
374 unsigned long pagesize;
376 if (large_page_bucket(i, sp_bits))
377 pagesize = 1UL << lp_bits;
379 pagesize = 1UL << sp_bits;
381 head->bs[i].elements_per_page
382 = elements_per_page(i / INTER_BUCKET_SPACE,
387 /* They start as all large pages. */
388 memset(head->pagesize, 0xFF, sizeof(head->pagesize));
389 /* FIXME: small pages for last bit? */
391 /* Split first page into small pages. */
392 assert(header_size < (1UL << lp_bits));
393 clear_bit(head->pagesize, 0);
395 /* Skip over page(s) used by header, add rest to free list */
396 for (i = align_up(header_size, (1 << sp_bits)) >> sp_bits;
397 i < SMALL_PAGES_PER_LARGE_PAGE;
399 ph = from_pgnum(head, i, sp_bits);
400 ph->elements_used = 0;
401 add_small_page_to_freelist(head, ph, sp_bits);
404 /* Add the rest of the pages as large pages. */
405 i = SMALL_PAGES_PER_LARGE_PAGE;
406 while ((i << sp_bits) + (1 << lp_bits) <= poolsize) {
407 ph = from_pgnum(head, i, sp_bits);
408 ph->elements_used = 0;
409 add_large_page_to_freelist(head, ph, sp_bits);
410 i += SMALL_PAGES_PER_LARGE_PAGE;
414 /* A large page worth of small pages are free: delete them from free list. */
415 static void del_large_from_small_free_list(struct header *head,
416 struct page_header *ph,
417 unsigned int sp_bits)
421 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
422 del_from_list(head, &head->small_free_list,
423 (void *)ph + (i << sp_bits),
428 static bool all_empty(struct header *head,
434 for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
435 struct page_header *ph = from_pgnum(head, pgnum + i, sp_bits);
436 if (ph->elements_used)
442 static void recombine_small_pages(struct header *head, unsigned long poolsize,
443 unsigned int sp_bits)
446 unsigned int lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
448 /* Look for small pages to coalesce, after first large page. */
449 for (i = SMALL_PAGES_PER_LARGE_PAGE;
450 i < (poolsize >> lp_bits) << BITS_FROM_SMALL_TO_LARGE_PAGE;
451 i += SMALL_PAGES_PER_LARGE_PAGE) {
452 /* Already a large page? */
453 if (test_bit(head->pagesize, i / SMALL_PAGES_PER_LARGE_PAGE))
455 if (all_empty(head, i, sp_bits)) {
456 struct page_header *ph = from_pgnum(head, i, sp_bits);
457 set_bit(head->pagesize,
458 i / SMALL_PAGES_PER_LARGE_PAGE);
459 del_large_from_small_free_list(head, ph, sp_bits);
460 add_large_page_to_freelist(head, ph, sp_bits);
465 static u16 get_large_page(struct header *head, unsigned long poolsize,
466 unsigned int sp_bits)
468 unsigned int lp_bits, page;
470 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
472 page = pop_from_list(head, &head->large_free_list, sp_bits);
476 recombine_small_pages(head, poolsize, sp_bits);
478 return pop_from_list(head, &head->large_free_list, sp_bits);
481 /* Returns small page. */
482 static unsigned long break_up_large_page(struct header *head,
483 unsigned int sp_bits,
488 clear_bit(head->pagesize, lpage >> BITS_FROM_SMALL_TO_LARGE_PAGE);
490 for (i = 1; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
491 struct page_header *ph = from_pgnum(head, lpage + i, sp_bits);
492 add_small_page_to_freelist(head, ph, sp_bits);
498 static u16 get_small_page(struct header *head, unsigned long poolsize,
499 unsigned int sp_bits)
503 ret = pop_from_list(head, &head->small_free_list, sp_bits);
506 ret = get_large_page(head, poolsize, sp_bits);
508 ret = break_up_large_page(head, sp_bits, ret);
512 void where_is_page(struct header *head, struct page_header *where,
513 unsigned int sp_bits)
515 struct page_header *pg;
516 unsigned long off, bucket,
517 num_buckets = max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE);
519 for (off = head->small_free_list; off; off = pg->next) {
520 pg = from_pgnum(head, off, sp_bits);
522 printf("It's in the small free list\n");
527 for (off = head->large_free_list; off; off = pg->next) {
528 pg = from_pgnum(head, off, sp_bits);
530 printf("It's in the large free list\n");
535 for (bucket = 0; bucket < num_buckets; bucket++) {
536 for (off = head->bs[bucket].page_list; off; off = pg->next) {
537 pg = from_pgnum(head, off, sp_bits);
539 printf("It's in %lu bucket page list\n", bucket);
544 for (off = head->bs[bucket].full_list; off; off = pg->next) {
545 pg = from_pgnum(head, off, sp_bits);
547 printf("It's in %lu bucket full list\n", bucket);
552 printf("It's nowhere!\n");
555 static bool huge_allocated(struct header *head, unsigned long offset)
558 struct huge_alloc *ha;
560 for (i = head->huge; i; i = ha->next) {
561 ha = (void *)((char *)head + i);
562 if (ha->off <= offset && ha->off + ha->len > offset)
568 /* They want something really big. Aim for contiguous pages (slow). */
569 static void *unlikely_func huge_alloc(void *pool, unsigned long poolsize,
570 unsigned long size, unsigned long align)
572 struct header *head = pool;
573 struct huge_alloc *ha;
574 unsigned long i, sp_bits, lp_bits, num, header_size;
576 sp_bits = small_page_bits(poolsize);
577 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
579 /* Allocate tracking structure optimistically. */
580 ha = alloc_get(pool, poolsize, sizeof(*ha), ALIGNOF(*ha));
584 /* First search for contiguous small pages... */
585 header_size = sizeof(*head) + sizeof(head->bs) * (max_bucket(lp_bits)-1);
588 for (i = (header_size + (1 << sp_bits) - 1) >> sp_bits;
589 i << sp_bits < poolsize;
591 struct page_header *pg;
592 unsigned long off = (i << sp_bits);
594 /* Skip over large pages. */
595 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
596 i += (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
600 /* Does this page meet alignment requirements? */
601 if (!num && off % align != 0)
604 /* FIXME: This makes us O(n^2). */
605 if (huge_allocated(head, off)) {
610 pg = (struct page_header *)((char *)head + off);
611 if (pg->elements_used) {
617 if (num << sp_bits >= size) {
620 /* Remove from free list. */
621 for (pgnum = i; pgnum > i - num; pgnum--) {
622 pg = from_pgnum(head, pgnum, sp_bits);
624 &head->small_free_list,
627 ha->off = (i - num + 1) << sp_bits;
628 ha->len = num << sp_bits;
633 /* Now search for large pages... */
634 recombine_small_pages(head, poolsize, sp_bits);
637 for (i = (header_size + (1 << lp_bits) - 1) >> lp_bits;
638 (i << lp_bits) < poolsize; i++) {
639 struct page_header *pg;
640 unsigned long off = (i << lp_bits);
642 /* Ignore small pages. */
643 if (!test_bit(head->pagesize, i))
646 /* Does this page meet alignment requirements? */
647 if (!num && off % align != 0)
650 /* FIXME: This makes us O(n^2). */
651 if (huge_allocated(head, off)) {
656 pg = (struct page_header *)((char *)head + off);
657 if (pg->elements_used) {
663 if (num << lp_bits >= size) {
666 /* Remove from free list. */
667 for (pgnum = i; pgnum > i - num; pgnum--) {
668 pg = from_pgnum(head, pgnum, lp_bits);
670 &head->large_free_list,
673 ha->off = (i - num + 1) << lp_bits;
674 ha->len = num << lp_bits;
679 /* Unable to satisfy: free huge alloc structure. */
680 alloc_free(pool, poolsize, ha);
684 add_to_huge_list(pool, ha);
685 return (char *)pool + ha->off;
688 static void unlikely_func huge_free(struct header *head,
689 unsigned long poolsize, void *free)
691 unsigned long i, off, pgnum, free_off = (char *)free - (char *)head;
692 unsigned int sp_bits, lp_bits;
693 struct huge_alloc *ha;
695 for (i = head->huge; i; i = ha->next) {
696 ha = (void *)((char *)head + i);
697 if (free_off == ha->off)
702 /* Free up all the pages, delete and free ha */
703 sp_bits = small_page_bits(poolsize);
704 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
705 pgnum = free_off >> sp_bits;
707 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
708 for (off = ha->off; off < ha->off + ha->len; off += 1 << lp_bits) {
709 add_large_page_to_freelist(head,
710 (void *)((char *)head + off),
714 for (off = ha->off; off < ha->off + ha->len; off += 1 << sp_bits) {
715 add_small_page_to_freelist(head,
716 (void *)((char *)head + off),
720 del_from_huge(head, ha);
721 alloc_free(head, poolsize, ha);
724 static unsigned long unlikely_func huge_size(struct header *head, void *p)
726 unsigned long i, off = (char *)p - (char *)head;
727 struct huge_alloc *ha;
729 for (i = head->huge; i; i = ha->next) {
730 ha = (void *)((char *)head + i);
731 if (off == ha->off) {
738 void *alloc_get(void *pool, unsigned long poolsize,
739 unsigned long size, unsigned long align)
741 struct header *head = pool;
744 struct bucket_state *bs;
745 struct page_header *ph;
746 unsigned int sp_bits;
748 if (poolsize < MIN_USEFUL_SIZE) {
749 return tiny_alloc_get(pool, poolsize, size, align);
752 size = align_up(size, align);
755 bucket = size_to_bucket(size);
757 sp_bits = small_page_bits(poolsize);
759 if (bucket >= max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE)) {
760 return huge_alloc(pool, poolsize, size, align);
763 bs = &head->bs[bucket];
765 if (!bs->page_list) {
766 struct page_header *ph;
768 if (large_page_bucket(bucket, sp_bits))
769 bs->page_list = get_large_page(head, poolsize,
772 bs->page_list = get_small_page(head, poolsize,
774 /* FIXME: Try large-aligned alloc? Header stuffing? */
775 if (unlikely(!bs->page_list))
777 ph = from_pgnum(head, bs->page_list, sp_bits);
779 ph->elements_used = 0;
781 memset(ph->used, 0, used_size(bs->elements_per_page));
784 ph = from_pgnum(head, bs->page_list, sp_bits);
786 i = find_free_bit(ph->used);
787 set_bit(ph->used, i);
790 /* check if this page is now full */
791 if (unlikely(ph->elements_used == bs->elements_per_page)) {
792 del_from_bucket_list(head, bs, ph, sp_bits);
793 add_to_bucket_full_list(head, bs, ph, sp_bits);
796 return (char *)ph + page_header_size(ph->bucket / INTER_BUCKET_SPACE,
797 bs->elements_per_page)
798 + i * bucket_to_size(bucket);
801 void alloc_free(void *pool, unsigned long poolsize, void *free)
803 struct header *head = pool;
804 struct bucket_state *bs;
805 unsigned int sp_bits;
806 unsigned long i, pgnum, pgoffset, offset = (char *)free - (char *)pool;
808 struct page_header *ph;
810 if (poolsize < MIN_USEFUL_SIZE) {
811 return tiny_alloc_free(pool, poolsize, free);
814 /* Get page header. */
815 sp_bits = small_page_bits(poolsize);
816 pgnum = offset >> sp_bits;
818 /* Big page? Round down further. */
819 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
821 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
825 /* Step back to page header. */
826 ph = from_pgnum(head, pgnum, sp_bits);
827 if ((void *)ph == free) {
828 huge_free(head, poolsize, free);
832 bs = &head->bs[ph->bucket];
833 pgoffset = offset - (pgnum << sp_bits)
834 - page_header_size(ph->bucket / INTER_BUCKET_SPACE,
835 bs->elements_per_page);
837 if (unlikely(ph->elements_used == bs->elements_per_page)) {
838 del_from_bucket_full_list(head, bs, ph, sp_bits);
839 add_to_bucket_list(head, bs, ph, sp_bits);
842 /* Which element are we? */
843 i = pgoffset / bucket_to_size(ph->bucket);
844 clear_bit(ph->used, i);
847 if (unlikely(ph->elements_used == 0)) {
848 bs = &head->bs[ph->bucket];
849 del_from_bucket_list(head, bs, ph, sp_bits);
851 add_small_page_to_freelist(head, ph, sp_bits);
853 add_large_page_to_freelist(head, ph, sp_bits);
857 unsigned long alloc_size(void *pool, unsigned long poolsize, void *p)
859 struct header *head = pool;
860 unsigned int pgnum, sp_bits;
861 unsigned long offset = (char *)p - (char *)pool;
862 struct page_header *ph;
864 if (poolsize < MIN_USEFUL_SIZE)
865 return tiny_alloc_size(pool, poolsize, p);
867 /* Get page header. */
868 sp_bits = small_page_bits(poolsize);
869 pgnum = offset >> sp_bits;
871 /* Big page? Round down further. */
872 if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE))
873 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
875 /* Step back to page header. */
876 ph = from_pgnum(head, pgnum, sp_bits);
878 return huge_size(head, p);
880 return bucket_to_size(ph->bucket);
883 /* Useful for gdb breakpoints. */
884 static bool check_fail(void)
889 static unsigned long count_bits(const unsigned long bitmap[],
892 unsigned long i, count = 0;
894 while (limit >= BITS_PER_LONG) {
895 count += popcount(bitmap[0]);
897 limit -= BITS_PER_LONG;
900 for (i = 0; i < limit; i++)
901 if (test_bit(bitmap, i))
906 static bool out_of_bounds(unsigned long pgnum,
907 unsigned int sp_bits,
908 unsigned long pagesize,
909 unsigned long poolsize)
911 if (((pgnum << sp_bits) >> sp_bits) != pgnum)
914 if ((pgnum << sp_bits) > poolsize)
917 return ((pgnum << sp_bits) + pagesize > poolsize);
920 static bool check_bucket(struct header *head,
921 unsigned long poolsize,
922 unsigned long pages[],
923 struct bucket_state *bs,
927 struct page_header *ph;
928 unsigned long taken, i, prev, pagesize, sp_bits, lp_bits;
930 sp_bits = small_page_bits(poolsize);
931 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
933 lp_bucket = large_page_bucket(bindex, sp_bits);
935 pagesize = 1UL << (lp_bucket ? lp_bits : sp_bits);
937 /* This many elements fit? */
938 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
939 bs->elements_per_page);
940 taken += bucket_to_size(bindex) * bs->elements_per_page;
941 if (taken > pagesize)
944 /* One more wouldn't fit? */
945 taken = page_header_size(bindex / INTER_BUCKET_SPACE,
946 bs->elements_per_page + 1);
947 taken += bucket_to_size(bindex) * (bs->elements_per_page + 1);
948 if (taken <= pagesize)
951 /* Walk used list. */
953 for (i = bs->page_list; i; i = ph->next) {
955 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
957 /* Wrong size page? */
958 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
961 /* Large page not on boundary? */
962 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
964 ph = from_pgnum(head, i, sp_bits);
965 /* Linked list corrupt? */
966 if (ph->prev != prev)
968 /* Already seen this page? */
969 if (test_bit(pages, i))
973 if (ph->elements_used == 0)
975 if (ph->elements_used >= bs->elements_per_page)
977 /* Used bits don't agree? */
978 if (ph->elements_used != count_bits(ph->used,
979 bs->elements_per_page))
982 if (ph->bucket != bindex)
987 /* Walk full list. */
989 for (i = bs->full_list; i; i = ph->next) {
991 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
993 /* Wrong size page? */
994 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
996 /* Large page not on boundary? */
997 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
999 ph = from_pgnum(head, i, sp_bits);
1000 /* Linked list corrupt? */
1001 if (ph->prev != prev)
1002 return check_fail();
1003 /* Already seen this page? */
1004 if (test_bit(pages, i))
1005 return check_fail();
1008 if (ph->elements_used != bs->elements_per_page)
1009 return check_fail();
1010 /* Used bits don't agree? */
1011 if (ph->elements_used != count_bits(ph->used,
1012 bs->elements_per_page))
1013 return check_fail();
1015 if (ph->bucket != bindex)
1016 return check_fail();
1022 bool alloc_check(void *pool, unsigned long poolsize)
1024 struct header *head = pool;
1025 unsigned long prev, i, lp_bits, sp_bits, header_size, num_buckets;
1026 struct page_header *ph;
1027 struct huge_alloc *ha;
1028 unsigned long pages[MAX_SMALL_PAGES / BITS_PER_LONG] = { 0 };
1030 if (poolsize < MIN_USEFUL_SIZE)
1031 return tiny_alloc_check(pool, poolsize);
1033 sp_bits = small_page_bits(poolsize);
1034 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1036 num_buckets = max_bucket(lp_bits);
1038 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1040 /* First, set all bits taken by header. */
1041 for (i = 0; i < header_size; i += (1UL << sp_bits))
1042 set_bit(pages, i >> sp_bits);
1044 /* Check small page free list. */
1046 for (i = head->small_free_list; i; i = ph->next) {
1048 if (out_of_bounds(i, sp_bits, 1 << sp_bits, poolsize))
1049 return check_fail();
1051 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1052 return check_fail();
1053 ph = from_pgnum(head, i, sp_bits);
1054 /* Linked list corrupt? */
1055 if (ph->prev != prev)
1056 return check_fail();
1057 /* Already seen this page? */
1058 if (test_bit(pages, i))
1059 return check_fail();
1064 /* Check large page free list. */
1066 for (i = head->large_free_list; i; i = ph->next) {
1068 if (out_of_bounds(i, sp_bits, 1 << lp_bits, poolsize))
1069 return check_fail();
1070 /* Not large page? */
1071 if (!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1072 return check_fail();
1073 /* Not page boundary? */
1074 if ((i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
1075 return check_fail();
1076 ph = from_pgnum(head, i, sp_bits);
1077 /* Linked list corrupt? */
1078 if (ph->prev != prev)
1079 return check_fail();
1080 /* Already seen this page? */
1081 if (test_bit(pages, i))
1082 return check_fail();
1087 /* Check the buckets. */
1088 for (i = 0; i < max_bucket(lp_bits); i++) {
1089 struct bucket_state *bs = &head->bs[i];
1091 if (!check_bucket(head, poolsize, pages, bs, i))
1095 /* Check the huge alloc list. */
1097 for (i = head->huge; i; i = ha->next) {
1098 unsigned long pgbits, j;
1101 if (i >= poolsize || i + sizeof(*ha) > poolsize)
1102 return check_fail();
1103 ha = (void *)((char *)head + i);
1105 /* Check contents of ha. */
1106 if (ha->off > poolsize || ha->off + ha->len > poolsize)
1107 return check_fail();
1109 /* Large or small page? */
1110 pgbits = test_bit(head->pagesize, ha->off >> lp_bits)
1111 ? lp_bits : sp_bits;
1113 /* Not page boundary? */
1114 if ((ha->off % (1UL << pgbits)) != 0)
1115 return check_fail();
1117 /* Not page length? */
1118 if ((ha->len % (1UL << pgbits)) != 0)
1119 return check_fail();
1121 /* Linked list corrupt? */
1122 if (ha->prev != prev)
1123 return check_fail();
1125 for (j = ha->off; j < ha->off + ha->len; j += (1 << sp_bits)) {
1126 /* Already seen this page? */
1127 if (test_bit(pages, j >> sp_bits))
1128 return check_fail();
1129 set_bit(pages, j >> sp_bits);
1135 /* Make sure every page accounted for. */
1136 for (i = 0; i < poolsize >> sp_bits; i++) {
1137 if (!test_bit(pages, i))
1138 return check_fail();
1139 if (test_bit(head->pagesize,
1140 i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
1141 /* Large page, skip rest. */
1142 i += SMALL_PAGES_PER_LARGE_PAGE - 1;
1149 static unsigned long print_overhead(FILE *out, const char *desc,
1150 unsigned long bytes,
1151 unsigned long poolsize)
1153 fprintf(out, "Overhead (%s): %lu bytes (%.3g%%)\n",
1154 desc, bytes, 100.0 * bytes / poolsize);
1158 static unsigned long count_list(struct header *head,
1160 unsigned int sp_bits,
1161 unsigned long *total_elems)
1163 struct page_header *p;
1164 unsigned long ret = 0;
1167 p = from_pgnum(head, pgnum, sp_bits);
1169 (*total_elems) += p->elements_used;
1176 static unsigned long visualize_bucket(FILE *out, struct header *head,
1177 unsigned int bucket,
1178 unsigned long poolsize,
1179 unsigned int sp_bits)
1181 unsigned long num_full, num_partial, num_pages, page_size,
1182 elems, hdr_min, hdr_size, elems_per_page, overhead = 0;
1184 elems_per_page = head->bs[bucket].elements_per_page;
1186 /* If we used byte-based bitmaps, we could get pg hdr to: */
1187 hdr_min = sizeof(struct page_header)
1188 - sizeof(((struct page_header *)0)->used)
1189 + align_up(elems_per_page, CHAR_BIT) / CHAR_BIT;
1190 hdr_size = page_header_size(bucket / INTER_BUCKET_SPACE,
1194 num_full = count_list(head, head->bs[bucket].full_list, sp_bits,
1196 num_partial = count_list(head, head->bs[bucket].page_list, sp_bits,
1198 num_pages = num_full + num_partial;
1202 fprintf(out, "Bucket %u (%lu bytes):"
1203 " %lu full, %lu partial = %lu elements\n",
1204 bucket, bucket_to_size(bucket), num_full, num_partial, elems);
1205 /* Strict requirement of page header size. */
1206 overhead += print_overhead(out, "page headers",
1207 hdr_min * num_pages, poolsize);
1208 /* Gap between minimal page header and actual start. */
1209 overhead += print_overhead(out, "page post-header alignments",
1210 (hdr_size - hdr_min) * num_pages, poolsize);
1211 /* Between last element and end of page. */
1212 page_size = (1 << sp_bits);
1213 if (large_page_bucket(bucket, sp_bits))
1214 page_size <<= BITS_FROM_SMALL_TO_LARGE_PAGE;
1216 overhead += print_overhead(out, "page tails",
1217 (page_size - (hdr_size
1219 * bucket_to_size(bucket))))
1220 * num_pages, poolsize);
1224 void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
1226 struct header *head = pool;
1227 unsigned long i, lp_bits, sp_bits, header_size, num_buckets, count,
1230 fprintf(out, "Pool %p size %lu: (%s allocator)\n", pool, poolsize,
1231 poolsize < MIN_USEFUL_SIZE ? "tiny" : "standard");
1233 if (poolsize < MIN_USEFUL_SIZE) {
1234 tiny_alloc_visualize(out, pool, poolsize);
1238 sp_bits = small_page_bits(poolsize);
1239 lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1241 num_buckets = max_bucket(lp_bits);
1242 header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1244 fprintf(out, "Large page size %lu, small page size %lu.\n",
1245 1UL << lp_bits, 1UL << sp_bits);
1246 overhead += print_overhead(out, "unused pool tail",
1247 poolsize % (1 << lp_bits), poolsize);
1248 fprintf(out, "Main header %lu bytes (%lu small pages).\n",
1249 header_size, align_up(header_size, 1 << sp_bits) >> sp_bits);
1250 overhead += print_overhead(out, "partial header page",
1251 align_up(header_size, 1 << sp_bits)
1252 - header_size, poolsize);
1253 /* Total large pages. */
1254 i = count_bits(head->pagesize, poolsize >> lp_bits);
1256 count = i - count_list(head, head->large_free_list, sp_bits, NULL);
1257 fprintf(out, "%lu/%lu large pages used (%.3g%%)\n",
1258 count, i, count ? 100.0 * count / i : 0.0);
1260 /* Total small pages. */
1261 i = ((poolsize >> lp_bits) - i) << BITS_FROM_SMALL_TO_LARGE_PAGE;
1263 count = i - count_list(head, head->small_free_list, sp_bits, NULL);
1264 fprintf(out, "%lu/%lu small pages used (%.3g%%)\n",
1265 count, i, count ? 100.0 * count / i : 0.0);
1267 /* Summary of each bucket. */
1268 fprintf(out, "%lu buckets:\n", num_buckets);
1269 for (i = 0; i < num_buckets; i++)
1270 overhead += visualize_bucket(out, head, i, poolsize, sp_bits);
1272 print_overhead(out, "total", overhead, poolsize);