#include <ccan/likely/likely.h>
#include <ccan/alignof/alignof.h>
#include <ccan/short_types/short_types.h>
+#include <ccan/compiler/compiler.h>
#include "config.h"
/*
*/
static unsigned long bucket_to_size(unsigned int bucket)
{
- unsigned long base = 1 << (bucket / INTER_BUCKET_SPACE);
+ unsigned long base = 1UL << (bucket / INTER_BUCKET_SPACE);
return base + ((bucket % INTER_BUCKET_SPACE)
<< (bucket / INTER_BUCKET_SPACE))
/ INTER_BUCKET_SPACE;
unsigned int base = fls(size/2);
unsigned long overshoot;
- overshoot = size - (1 << base);
+ overshoot = size - (1UL << base);
return base * INTER_BUCKET_SPACE
- + ((overshoot * INTER_BUCKET_SPACE + (1 << base)-1) >> base);
+ + ((overshoot * INTER_BUCKET_SPACE + (1UL << base)-1) >> base);
}
static unsigned int small_page_bits(unsigned long poolsize)
{
- return fls(poolsize / MAX_SMALL_PAGES / 2);
+ return fls(poolsize / MAX_SMALL_PAGES - 1);
}
static struct page_header *from_pgnum(struct header *head,
size = sizeof(struct page_header)
- sizeof(((struct page_header *)0)->used)
+ used_size(num_elements);
- return align_up(size, 1 << align_bits);
+ return align_up(size, 1UL << align_bits);
}
static void add_to_list(struct header *head,
static void clear_bit(unsigned long bitmap[], unsigned int off)
{
- bitmap[off / BITS_PER_LONG] &= ~(1 << (off % BITS_PER_LONG));
+ bitmap[off / BITS_PER_LONG] &= ~(1UL << (off % BITS_PER_LONG));
}
static bool test_bit(const unsigned long bitmap[], unsigned int off)
{
- return bitmap[off / BITS_PER_LONG] & (1 << (off % BITS_PER_LONG));
+ return bitmap[off / BITS_PER_LONG] & (1UL << (off % BITS_PER_LONG));
}
static void set_bit(unsigned long bitmap[], unsigned int off)
{
- bitmap[off / BITS_PER_LONG] |= (1 << (off % BITS_PER_LONG));
+ bitmap[off / BITS_PER_LONG] |= (1UL << (off % BITS_PER_LONG));
}
/* There must be a bit to be found. */
unsigned long num, overhead;
/* First approximation: no extra room for bitmap. */
- overhead = align_up(sizeof(struct page_header), 1 << align_bits);
+ overhead = align_up(sizeof(struct page_header), 1UL << align_bits);
num = (psize - overhead) / esize;
while (page_header_size(align_bits, num) + esize * num > psize)
clear_bit(head->pagesize, 0);
/* Skip over page(s) used by header, add rest to free list */
- for (i = align_up(header_size, (1 << sp_bits)) >> sp_bits;
+ for (i = align_up(header_size, (1UL << sp_bits)) >> sp_bits;
i < SMALL_PAGES_PER_LARGE_PAGE;
i++) {
ph = from_pgnum(head, i, sp_bits);
/* Add the rest of the pages as large pages. */
i = SMALL_PAGES_PER_LARGE_PAGE;
- while ((i << sp_bits) + (1 << lp_bits) <= poolsize) {
+ while ((i << sp_bits) + (1UL << lp_bits) <= poolsize) {
+ assert(i < MAX_SMALL_PAGES);
ph = from_pgnum(head, i, sp_bits);
ph->elements_used = 0;
add_large_page_to_freelist(head, ph, sp_bits);
static u16 get_large_page(struct header *head, unsigned long poolsize,
unsigned int sp_bits)
{
- unsigned int lp_bits, page;
-
- lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
+ unsigned int page;
page = pop_from_list(head, &head->large_free_list, sp_bits);
if (likely(page))
for (i = 1; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
struct page_header *ph = from_pgnum(head, lpage + i, sp_bits);
+ /* Initialize this: huge_alloc reads it. */
+ ph->elements_used = 0;
add_small_page_to_freelist(head, ph, sp_bits);
}
return ret;
}
-void where_is_page(struct header *head, struct page_header *where,
- unsigned int sp_bits)
-{
- struct page_header *pg;
- unsigned long off, bucket,
- num_buckets = max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE);
-
- for (off = head->small_free_list; off; off = pg->next) {
- pg = from_pgnum(head, off, sp_bits);
- if (pg == where) {
- printf("It's in the small free list\n");
- return;
- }
- }
-
- for (off = head->large_free_list; off; off = pg->next) {
- pg = from_pgnum(head, off, sp_bits);
- if (pg == where) {
- printf("It's in the large free list\n");
- return;
- }
- }
-
- for (bucket = 0; bucket < num_buckets; bucket++) {
- for (off = head->bs[bucket].page_list; off; off = pg->next) {
- pg = from_pgnum(head, off, sp_bits);
- if (pg == where) {
- printf("It's in %lu bucket page list\n", bucket);
- return;
- }
- }
-
- for (off = head->bs[bucket].full_list; off; off = pg->next) {
- pg = from_pgnum(head, off, sp_bits);
- if (pg == where) {
- printf("It's in %lu bucket full list\n", bucket);
- return;
- }
- }
- }
- printf("It's nowhere!\n");
-}
-
static bool huge_allocated(struct header *head, unsigned long offset)
{
unsigned long i;
}
/* They want something really big. Aim for contiguous pages (slow). */
-static void *unlikely_func huge_alloc(void *pool, unsigned long poolsize,
- unsigned long size, unsigned long align)
+static COLD void *huge_alloc(void *pool, unsigned long poolsize,
+ unsigned long size, unsigned long align)
{
struct header *head = pool;
struct huge_alloc *ha;
header_size = sizeof(*head) + sizeof(head->bs) * (max_bucket(lp_bits)-1);
num = 0;
- for (i = (header_size + (1 << sp_bits) - 1) >> sp_bits;
+ for (i = (header_size + (1UL << sp_bits) - 1) >> sp_bits;
i << sp_bits < poolsize;
i++) {
struct page_header *pg;
/* Skip over large pages. */
if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
- i += (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
+ i += (1UL << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
continue;
}
recombine_small_pages(head, poolsize, sp_bits);
num = 0;
- for (i = (header_size + (1 << lp_bits) - 1) >> lp_bits;
+ for (i = (header_size + (1UL << lp_bits) - 1) >> lp_bits;
(i << lp_bits) < poolsize; i++) {
struct page_header *pg;
unsigned long off = (i << lp_bits);
return (char *)pool + ha->off;
}
-static void unlikely_func huge_free(struct header *head,
- unsigned long poolsize, void *free)
+static COLD void
+huge_free(struct header *head, unsigned long poolsize, void *free)
{
unsigned long i, off, pgnum, free_off = (char *)free - (char *)head;
unsigned int sp_bits, lp_bits;
pgnum = free_off >> sp_bits;
if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
- for (off = ha->off; off < ha->off + ha->len; off += 1 << lp_bits) {
+ for (off = ha->off;
+ off < ha->off + ha->len;
+ off += 1UL << lp_bits) {
add_large_page_to_freelist(head,
(void *)((char *)head + off),
sp_bits);
}
} else {
- for (off = ha->off; off < ha->off + ha->len; off += 1 << sp_bits) {
+ for (off = ha->off;
+ off < ha->off + ha->len;
+ off += 1UL << sp_bits) {
add_small_page_to_freelist(head,
(void *)((char *)head + off),
sp_bits);
alloc_free(head, poolsize, ha);
}
-static unsigned long unlikely_func huge_size(struct header *head, void *p)
+static COLD unsigned long huge_size(struct header *head, void *p)
{
unsigned long i, off = (char *)p - (char *)head;
struct huge_alloc *ha;
prev = 0;
for (i = head->small_free_list; i; i = ph->next) {
/* Bad pointer? */
- if (out_of_bounds(i, sp_bits, 1 << sp_bits, poolsize))
+ if (out_of_bounds(i, sp_bits, 1UL << sp_bits, poolsize))
return check_fail();
/* Large page? */
if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
prev = 0;
for (i = head->large_free_list; i; i = ph->next) {
/* Bad pointer? */
- if (out_of_bounds(i, sp_bits, 1 << lp_bits, poolsize))
+ if (out_of_bounds(i, sp_bits, 1UL << lp_bits, poolsize))
return check_fail();
/* Not large page? */
if (!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
if (ha->prev != prev)
return check_fail();
- for (j = ha->off; j < ha->off + ha->len; j += (1 << sp_bits)) {
+ for (j = ha->off; j < ha->off + ha->len; j += (1UL<<sp_bits)) {
/* Already seen this page? */
if (test_bit(pages, j >> sp_bits))
return check_fail();
overhead += print_overhead(out, "page post-header alignments",
(hdr_size - hdr_min) * num_pages, poolsize);
/* Between last element and end of page. */
- page_size = (1 << sp_bits);
+ page_size = (1UL << sp_bits);
if (large_page_bucket(bucket, sp_bits))
page_size <<= BITS_FROM_SMALL_TO_LARGE_PAGE;
fprintf(out, "Large page size %lu, small page size %lu.\n",
1UL << lp_bits, 1UL << sp_bits);
overhead += print_overhead(out, "unused pool tail",
- poolsize % (1 << lp_bits), poolsize);
+ poolsize % (1UL << lp_bits), poolsize);
fprintf(out, "Main header %lu bytes (%lu small pages).\n",
- header_size, align_up(header_size, 1 << sp_bits) >> sp_bits);
+ header_size, align_up(header_size, 1UL << sp_bits) >> sp_bits);
overhead += print_overhead(out, "partial header page",
- align_up(header_size, 1 << sp_bits)
+ align_up(header_size, 1UL << sp_bits)
- header_size, poolsize);
/* Total large pages. */
i = count_bits(head->pagesize, poolsize >> lp_bits);
projects / ccan / blobdiff