git.ozlabs.org Git - ccan/blob - ccan/antithread/alloc/alloc.c

   1 /* Licensed under LGPLv2.1+ - see LICENSE file for details */
   2 #include <unistd.h>
   3 #include <stdint.h>
   4 #include <string.h>
   5 #include <limits.h>
   6 #include <assert.h>
   7 #include <stdlib.h>
   8 #include "alloc.h"
   9 #include "bitops.h"
  10 #include "tiny.h"
  11 #include <ccan/build_assert/build_assert.h>
  12 #include <ccan/likely/likely.h>
  13 #include <ccan/alignof/alignof.h>
  14 #include <ccan/short_types/short_types.h>
  15 #include <ccan/compiler/compiler.h>
  16 #include "config.h"
  17
  18 /*
  19    Inspired by (and parts taken from) Andrew Tridgell's alloc_mmap:
  20    http://samba.org/~tridge/junkcode/alloc_mmap/
  21
  22    Copyright (C) Andrew Tridgell 2007
  23
  24    This library is free software; you can redistribute it and/or
  25    modify it under the terms of the GNU Lesser General Public
  26    License as published by the Free Software Foundation; either
  27    version 2 of the License, or (at your option) any later version.
  28
  29    This library is distributed in the hope that it will be useful,
  30    but WITHOUT ANY WARRANTY; without even the implied warranty of
  31    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  32    Lesser General Public License for more details.
  33
  34    You should have received a copy of the GNU Lesser General Public
  35    License along with this library; if not, write to the Free Software
  36    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  37  */
  38
  39 /* We divide the pool into this many large pages (nearest power of 2) */
  40 #define MAX_LARGE_PAGES (256UL)
  41
  42 /* 32 small pages == 1 large page. */
  43 #define BITS_FROM_SMALL_TO_LARGE_PAGE 5
  44
  45 #define MAX_SMALL_PAGES (MAX_LARGE_PAGES << BITS_FROM_SMALL_TO_LARGE_PAGE)
  46
  47 /* Smallest pool size for this scheme: 128-byte small pages.  That's
  48  * 9/13% overhead for 32/64 bit. */
  49 #define MIN_USEFUL_SIZE (MAX_SMALL_PAGES * 128)
  50
  51 /* Every 4 buckets, we jump up a power of 2. ...8 10 12 14 16 20 24 28 32... */
  52 #define INTER_BUCKET_SPACE 4
  53
  54 #define SMALL_PAGES_PER_LARGE_PAGE (1 << BITS_FROM_SMALL_TO_LARGE_PAGE)
  55
  56 /* FIXME: Figure this out properly. */
  57 #define MAX_SIZE (1 << 30)
  58
  59 /* How few object to fit in a page before using a larger one? (8) */
  60 #define MAX_PAGE_OBJECT_ORDER   3
  61
  62 #define BITS_PER_LONG (sizeof(long) * CHAR_BIT)
  63
  64 struct bucket_state {
  65         u32 elements_per_page;
  66         u16 page_list;
  67         u16 full_list;
  68 };
  69
  70 struct header {
  71         /* Bitmap of which pages are large. */
  72         unsigned long pagesize[MAX_LARGE_PAGES / BITS_PER_LONG];
  73
  74         /* List of unused small/large pages. */
  75         u16 small_free_list;
  76         u16 large_free_list;
  77
  78         /* List of huge allocs. */
  79         unsigned long huge;
  80
  81         /* This is less defined: we have two buckets for each power of 2 */
  82         struct bucket_state bs[1];
  83 };
  84
  85 struct huge_alloc {
  86         unsigned long next, prev;
  87         unsigned long off, len;
  88 };
  89
  90 struct page_header {
  91         u16 next, prev;
  92         /* FIXME: We can just count all-0 and all-1 used[] elements. */
  93         unsigned elements_used : 25;
  94         unsigned bucket : 7;
  95         unsigned long used[1]; /* One bit per element. */
  96 };
  97
  98 /*
  99  * Every 4 buckets, the size doubles.
 100  * Between buckets, sizes increase linearly.
 101  *
 102  * eg. bucket 40 = 2^10                 = 1024
 103  *     bucket 41 = 2^10 + 2^10*4        = 1024 + 256
 104  *     bucket 42 = 2^10 + 2^10*4        = 1024 + 512
 105  *     bucket 43 = 2^10 + 2^10*4        = 1024 + 768
 106  *     bucket 45 = 2^11                 = 2048
 107  *
 108  * Care is taken to handle low numbered buckets, at cost of overflow.
 109  */
 110 static unsigned long bucket_to_size(unsigned int bucket)
 111 {
 112         unsigned long base = 1UL << (bucket / INTER_BUCKET_SPACE);
 113         return base + ((bucket % INTER_BUCKET_SPACE)
 114                        << (bucket / INTER_BUCKET_SPACE))
 115                 / INTER_BUCKET_SPACE;
 116 }
 117
 118 /*
 119  * Say size is 10.
 120  *   fls(size/2) == 3.  1 << 3 == 8, so we're 2 too large, out of a possible
 121  * 8 too large.  That's 1/4 of the way to the next power of 2 == 1 bucket.
 122  *
 123  * We make sure we round up.  Note that this fails on 32 bit at size
 124  * 1879048193 (around bucket 120).
 125  */
 126 static unsigned int size_to_bucket(unsigned long size)
 127 {
 128         unsigned int base = afls(size/2);
 129         unsigned long overshoot;
 130
 131         overshoot = size - (1UL << base);
 132         return base * INTER_BUCKET_SPACE
 133                 + ((overshoot * INTER_BUCKET_SPACE + (1UL << base)-1) >> base);
 134 }
 135
 136 static unsigned int small_page_bits(unsigned long poolsize)
 137 {
 138         return afls(poolsize / MAX_SMALL_PAGES - 1);
 139 }
 140
 141 static struct page_header *from_pgnum(struct header *head,
 142                                       unsigned long pgnum,
 143                                       unsigned sp_bits)
 144 {
 145         return (struct page_header *)((char *)head + (pgnum << sp_bits));
 146 }
 147
 148 static u16 to_pgnum(struct header *head, void *p, unsigned sp_bits)
 149 {
 150         return ((char *)p - (char *)head) >> sp_bits;
 151 }
 152
 153 static size_t used_size(unsigned int num_elements)
 154 {
 155         return align_up(num_elements, BITS_PER_LONG) / CHAR_BIT;
 156 }
 157
 158 /*
 159  * We always align the first entry to the lower power of 2.
 160  * eg. the 12-byte bucket gets 8-byte aligned.  The 4096-byte bucket
 161  * gets 4096-byte aligned.
 162  */
 163 static unsigned long page_header_size(unsigned int align_bits,
 164                                       unsigned long num_elements)
 165 {
 166         unsigned long size;
 167
 168         size = sizeof(struct page_header)
 169                 - sizeof(((struct page_header *)0)->used)
 170                 + used_size(num_elements);
 171         return align_up(size, 1UL << align_bits);
 172 }
 173
 174 static void add_to_list(struct header *head,
 175                         u16 *list, struct page_header *ph, unsigned sp_bits)
 176 {
 177         unsigned long h = *list, offset = to_pgnum(head, ph, sp_bits);
 178
 179         ph->next = h;
 180         if (h) {
 181                 struct page_header *prev = from_pgnum(head, h, sp_bits);
 182                 assert(prev->prev == 0);
 183                 prev->prev = offset;
 184         }
 185         *list = offset;
 186         ph->prev = 0;
 187 }
 188
 189 static void del_from_list(struct header *head,
 190                           u16 *list, struct page_header *ph, unsigned sp_bits)
 191 {
 192         /* Front of list? */
 193         if (ph->prev == 0) {
 194                 *list = ph->next;
 195         } else {
 196                 struct page_header *prev = from_pgnum(head, ph->prev, sp_bits);
 197                 prev->next = ph->next;
 198         }
 199         if (ph->next != 0) {
 200                 struct page_header *next = from_pgnum(head, ph->next, sp_bits);
 201                 next->prev = ph->prev;
 202         }
 203 }
 204
 205 static u16 pop_from_list(struct header *head,
 206                                    u16 *list,
 207                                    unsigned int sp_bits)
 208 {
 209         u16 h = *list;
 210         struct page_header *ph = from_pgnum(head, h, sp_bits);
 211
 212         if (likely(h)) {
 213                 *list = ph->next;
 214                 if (*list)
 215                         from_pgnum(head, *list, sp_bits)->prev = 0;
 216         }
 217         return h;
 218 }
 219
 220 static void add_to_huge_list(struct header *head, struct huge_alloc *ha)
 221 {
 222         unsigned long h = head->huge;
 223         unsigned long offset = (char *)ha - (char *)head;
 224
 225         ha->next = h;
 226         if (h) {
 227                 struct huge_alloc *prev = (void *)((char *)head + h);
 228                 assert(prev->prev == 0);
 229                 prev->prev = offset;
 230         }
 231         head->huge = offset;
 232         ha->prev = 0;
 233 }
 234
 235 static void del_from_huge(struct header *head, struct huge_alloc *ha)
 236 {
 237         /* Front of list? */
 238         if (ha->prev == 0) {
 239                 head->huge = ha->next;
 240         } else {
 241                 struct huge_alloc *prev = (void *)((char *)head + ha->prev);
 242                 prev->next = ha->next;
 243         }
 244         if (ha->next != 0) {
 245                 struct huge_alloc *next = (void *)((char *)head + ha->next);
 246                 next->prev = ha->prev;
 247         }
 248 }
 249
 250 static void add_small_page_to_freelist(struct header *head,
 251                                        struct page_header *ph,
 252                                        unsigned int sp_bits)
 253 {
 254         add_to_list(head, &head->small_free_list, ph, sp_bits);
 255 }
 256
 257 static void add_large_page_to_freelist(struct header *head,
 258                                        struct page_header *ph,
 259                                        unsigned int sp_bits)
 260 {
 261         add_to_list(head, &head->large_free_list, ph, sp_bits);
 262 }
 263
 264 static void add_to_bucket_list(struct header *head,
 265                                struct bucket_state *bs,
 266                                struct page_header *ph,
 267                                unsigned int sp_bits)
 268 {
 269         add_to_list(head, &bs->page_list, ph, sp_bits);
 270 }
 271
 272 static void del_from_bucket_list(struct header *head,
 273                                  struct bucket_state *bs,
 274                                  struct page_header *ph,
 275                                  unsigned int sp_bits)
 276 {
 277         del_from_list(head, &bs->page_list, ph, sp_bits);
 278 }
 279
 280 static void del_from_bucket_full_list(struct header *head,
 281                                       struct bucket_state *bs,
 282                                       struct page_header *ph,
 283                                       unsigned int sp_bits)
 284 {
 285         del_from_list(head, &bs->full_list, ph, sp_bits);
 286 }
 287
 288 static void add_to_bucket_full_list(struct header *head,
 289                                     struct bucket_state *bs,
 290                                     struct page_header *ph,
 291                                     unsigned int sp_bits)
 292 {
 293         add_to_list(head, &bs->full_list, ph, sp_bits);
 294 }
 295
 296 static void clear_bit(unsigned long bitmap[], unsigned int off)
 297 {
 298         bitmap[off / BITS_PER_LONG] &= ~(1UL << (off % BITS_PER_LONG));
 299 }
 300
 301 static bool test_bit(const unsigned long bitmap[], unsigned int off)
 302 {
 303         return bitmap[off / BITS_PER_LONG] & (1UL << (off % BITS_PER_LONG));
 304 }
 305
 306 static void set_bit(unsigned long bitmap[], unsigned int off)
 307 {
 308         bitmap[off / BITS_PER_LONG] |= (1UL << (off % BITS_PER_LONG));
 309 }
 310
 311 /* There must be a bit to be found. */
 312 static unsigned int find_free_bit(const unsigned long bitmap[])
 313 {
 314         unsigned int i;
 315
 316         for (i = 0; bitmap[i] == -1UL; i++);
 317         return (i*BITS_PER_LONG) + affsl(~bitmap[i]) - 1;
 318 }
 319
 320 /* How many elements can we fit in a page? */
 321 static unsigned long elements_per_page(unsigned long align_bits,
 322                                        unsigned long esize,
 323                                        unsigned long psize)
 324 {
 325         unsigned long num, overhead;
 326
 327         /* First approximation: no extra room for bitmap. */
 328         overhead = align_up(sizeof(struct page_header), 1UL << align_bits);
 329         num = (psize - overhead) / esize;
 330
 331         while (page_header_size(align_bits, num) + esize * num > psize)
 332                 num--;
 333         return num;
 334 }
 335
 336 static bool large_page_bucket(unsigned int bucket, unsigned int sp_bits)
 337 {
 338         unsigned long max_smallsize;
 339
 340         /* Note: this doesn't take into account page header. */
 341         max_smallsize = (1UL << sp_bits) >> MAX_PAGE_OBJECT_ORDER;
 342
 343         return bucket_to_size(bucket) > max_smallsize;
 344 }
 345
 346 static unsigned int max_bucket(unsigned int lp_bits)
 347 {
 348         return (lp_bits - MAX_PAGE_OBJECT_ORDER) * INTER_BUCKET_SPACE;
 349 }
 350
 351 void alloc_init(void *pool, unsigned long poolsize)
 352 {
 353         struct header *head = pool;
 354         struct page_header *ph;
 355         unsigned int lp_bits, sp_bits, num_buckets;
 356         unsigned long header_size, i;
 357
 358         if (poolsize < MIN_USEFUL_SIZE) {
 359                 tiny_alloc_init(pool, poolsize);
 360                 return;
 361         }
 362
 363         /* We rely on page numbers fitting in 16 bit. */
 364         BUILD_ASSERT(MAX_SMALL_PAGES < 65536);
 365
 366         sp_bits = small_page_bits(poolsize);
 367         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 368
 369         num_buckets = max_bucket(lp_bits);
 370
 371         head = pool;
 372         header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
 373
 374         memset(head, 0, header_size);
 375         for (i = 0; i < num_buckets; i++) {
 376                 unsigned long pagesize;
 377
 378                 if (large_page_bucket(i, sp_bits))
 379                         pagesize = 1UL << lp_bits;
 380                 else
 381                         pagesize = 1UL << sp_bits;
 382
 383                 head->bs[i].elements_per_page
 384                         = elements_per_page(i / INTER_BUCKET_SPACE,
 385                                             bucket_to_size(i),
 386                                             pagesize);
 387         }
 388
 389         /* They start as all large pages. */
 390         memset(head->pagesize, 0xFF, sizeof(head->pagesize));
 391         /* FIXME: small pages for last bit? */
 392
 393         /* Split first page into small pages. */
 394         assert(header_size < (1UL << lp_bits));
 395         clear_bit(head->pagesize, 0);
 396
 397         /* Skip over page(s) used by header, add rest to free list */
 398         for (i = align_up(header_size, (1UL << sp_bits)) >> sp_bits;
 399              i < SMALL_PAGES_PER_LARGE_PAGE;
 400              i++) {
 401                 ph = from_pgnum(head, i, sp_bits);
 402                 ph->elements_used = 0;
 403                 add_small_page_to_freelist(head, ph, sp_bits);
 404         }
 405
 406         /* Add the rest of the pages as large pages. */
 407         i = SMALL_PAGES_PER_LARGE_PAGE;
 408         while ((i << sp_bits) + (1UL << lp_bits) <= poolsize) {
 409                 assert(i < MAX_SMALL_PAGES);
 410                 ph = from_pgnum(head, i, sp_bits);
 411                 ph->elements_used = 0;
 412                 add_large_page_to_freelist(head, ph, sp_bits);
 413                 i += SMALL_PAGES_PER_LARGE_PAGE;
 414         }
 415 }
 416
 417 /* A large page worth of small pages are free: delete them from free list. */
 418 static void del_large_from_small_free_list(struct header *head,
 419                                            struct page_header *ph,
 420                                            unsigned int sp_bits)
 421 {
 422         unsigned long i;
 423
 424         for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
 425                 del_from_list(head, &head->small_free_list,
 426                               (struct page_header *)((char *)ph
 427                                                      + (i << sp_bits)),
 428                               sp_bits);
 429         }
 430 }
 431
 432 static bool all_empty(struct header *head,
 433                       unsigned long pgnum,
 434                       unsigned sp_bits)
 435 {
 436         unsigned long i;
 437
 438         for (i = 0; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
 439                 struct page_header *ph = from_pgnum(head, pgnum + i, sp_bits);
 440                 if (ph->elements_used)
 441                         return false;
 442         }
 443         return true;
 444 }
 445
 446 static void recombine_small_pages(struct header *head, unsigned long poolsize,
 447                                   unsigned int sp_bits)
 448 {
 449         unsigned long i;
 450         unsigned int lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 451
 452         /* Look for small pages to coalesce, after first large page. */
 453         for (i = SMALL_PAGES_PER_LARGE_PAGE;
 454              i < (poolsize >> lp_bits) << BITS_FROM_SMALL_TO_LARGE_PAGE;
 455              i += SMALL_PAGES_PER_LARGE_PAGE) {
 456                 /* Already a large page? */
 457                 if (test_bit(head->pagesize, i / SMALL_PAGES_PER_LARGE_PAGE))
 458                         continue;
 459                 if (all_empty(head, i, sp_bits)) {
 460                         struct page_header *ph = from_pgnum(head, i, sp_bits);
 461                         set_bit(head->pagesize,
 462                                 i / SMALL_PAGES_PER_LARGE_PAGE);
 463                         del_large_from_small_free_list(head, ph, sp_bits);
 464                         add_large_page_to_freelist(head, ph, sp_bits);
 465                 }
 466         }
 467 }
 468
 469 static u16 get_large_page(struct header *head, unsigned long poolsize,
 470                           unsigned int sp_bits)
 471 {
 472         unsigned int page;
 473
 474         page = pop_from_list(head, &head->large_free_list, sp_bits);
 475         if (likely(page))
 476                 return page;
 477
 478         recombine_small_pages(head, poolsize, sp_bits);
 479
 480         return pop_from_list(head, &head->large_free_list, sp_bits);
 481 }
 482
 483 /* Returns small page. */
 484 static unsigned long break_up_large_page(struct header *head,
 485                                          unsigned int sp_bits,
 486                                          u16 lpage)
 487 {
 488         unsigned int i;
 489
 490         clear_bit(head->pagesize, lpage >> BITS_FROM_SMALL_TO_LARGE_PAGE);
 491
 492         for (i = 1; i < SMALL_PAGES_PER_LARGE_PAGE; i++) {
 493                 struct page_header *ph = from_pgnum(head, lpage + i, sp_bits);
 494                 /* Initialize this: huge_alloc reads it. */
 495                 ph->elements_used = 0;
 496                 add_small_page_to_freelist(head, ph, sp_bits);
 497         }
 498
 499         return lpage;
 500 }
 501
 502 static u16 get_small_page(struct header *head, unsigned long poolsize,
 503                           unsigned int sp_bits)
 504 {
 505         u16 ret;
 506
 507         ret = pop_from_list(head, &head->small_free_list, sp_bits);
 508         if (likely(ret))
 509                 return ret;
 510         ret = get_large_page(head, poolsize, sp_bits);
 511         if (likely(ret))
 512                 ret = break_up_large_page(head, sp_bits, ret);
 513         return ret;
 514 }
 515
 516 static bool huge_allocated(struct header *head, unsigned long offset)
 517 {
 518         unsigned long i;
 519         struct huge_alloc *ha;
 520
 521         for (i = head->huge; i; i = ha->next) {
 522                 ha = (void *)((char *)head + i);
 523                 if (ha->off <= offset && ha->off + ha->len > offset)
 524                         return true;
 525         }
 526         return false;
 527 }
 528
 529 /* They want something really big.  Aim for contiguous pages (slow). */
 530 static COLD void *huge_alloc(void *pool, unsigned long poolsize,
 531                              unsigned long size, unsigned long align)
 532 {
 533         struct header *head = pool;
 534         struct huge_alloc *ha;
 535         unsigned long i, sp_bits, lp_bits, num, header_size;
 536
 537         sp_bits = small_page_bits(poolsize);
 538         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 539
 540         /* Allocate tracking structure optimistically. */
 541         ha = alloc_get(pool, poolsize, sizeof(*ha), ALIGNOF(*ha));
 542         if (!ha)
 543                 return NULL;
 544
 545         /* First search for contiguous small pages... */
 546         header_size = sizeof(*head) + sizeof(head->bs) * (max_bucket(lp_bits)-1);
 547
 548         num = 0;
 549         for (i = (header_size + (1UL << sp_bits) - 1) >> sp_bits;
 550              i << sp_bits < poolsize;
 551              i++) {
 552                 struct page_header *pg;
 553                 unsigned long off = (i << sp_bits);
 554
 555                 /* Skip over large pages. */
 556                 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
 557                         i += (1UL << BITS_FROM_SMALL_TO_LARGE_PAGE)-1;
 558                         continue;
 559                 }
 560
 561                 /* Does this page meet alignment requirements? */
 562                 if (!num && off % align != 0)
 563                         continue;
 564
 565                 /* FIXME: This makes us O(n^2). */
 566                 if (huge_allocated(head, off)) {
 567                         num = 0;
 568                         continue;
 569                 }
 570
 571                 pg = (struct page_header *)((char *)head + off);
 572                 if (pg->elements_used) {
 573                         num = 0;
 574                         continue;
 575                 }
 576
 577                 num++;
 578                 if (num << sp_bits >= size) {
 579                         unsigned long pgnum;
 580
 581                         /* Remove from free list. */
 582                         for (pgnum = i; pgnum > i - num; pgnum--) {
 583                                 pg = from_pgnum(head, pgnum, sp_bits);
 584                                 del_from_list(head,
 585                                               &head->small_free_list,
 586                                               pg, sp_bits);
 587                         }
 588                         ha->off = (i - num + 1) << sp_bits;
 589                         ha->len = num << sp_bits;
 590                         goto done;
 591                 }
 592         }
 593
 594         /* Now search for large pages... */
 595         recombine_small_pages(head, poolsize, sp_bits);
 596
 597         num = 0;
 598         for (i = (header_size + (1UL << lp_bits) - 1) >> lp_bits;
 599              (i << lp_bits) < poolsize; i++) {
 600                 struct page_header *pg;
 601                 unsigned long off = (i << lp_bits);
 602
 603                 /* Ignore small pages. */
 604                 if (!test_bit(head->pagesize, i))
 605                         continue;
 606
 607                 /* Does this page meet alignment requirements? */
 608                 if (!num && off % align != 0)
 609                         continue;
 610
 611                 /* FIXME: This makes us O(n^2). */
 612                 if (huge_allocated(head, off)) {
 613                         num = 0;
 614                         continue;
 615                 }
 616
 617                 pg = (struct page_header *)((char *)head + off);
 618                 if (pg->elements_used) {
 619                         num = 0;
 620                         continue;
 621                 }
 622
 623                 num++;
 624                 if (num << lp_bits >= size) {
 625                         unsigned long pgnum;
 626
 627                         /* Remove from free list. */
 628                         for (pgnum = i; pgnum > i - num; pgnum--) {
 629                                 pg = from_pgnum(head, pgnum, lp_bits);
 630                                 del_from_list(head,
 631                                               &head->large_free_list,
 632                                               pg, sp_bits);
 633                         }
 634                         ha->off = (i - num + 1) << lp_bits;
 635                         ha->len = num << lp_bits;
 636                         goto done;
 637                 }
 638         }
 639
 640         /* Unable to satisfy: free huge alloc structure. */
 641         alloc_free(pool, poolsize, ha);
 642         return NULL;
 643
 644 done:
 645         add_to_huge_list(pool, ha);
 646         return (char *)pool + ha->off;
 647 }
 648
 649 static COLD void
 650 huge_free(struct header *head, unsigned long poolsize, void *free)
 651 {
 652         unsigned long i, off, pgnum, free_off = (char *)free - (char *)head;
 653         unsigned int sp_bits, lp_bits;
 654         struct huge_alloc *ha;
 655
 656         for (i = head->huge; i; i = ha->next) {
 657                 ha = (void *)((char *)head + i);
 658                 if (free_off == ha->off)
 659                         break;
 660         }
 661         assert(i);
 662
 663         /* Free up all the pages, delete and free ha */
 664         sp_bits = small_page_bits(poolsize);
 665         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 666         pgnum = free_off >> sp_bits;
 667
 668         if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
 669                 for (off = ha->off;
 670                      off < ha->off + ha->len;
 671                      off += 1UL << lp_bits) {
 672                         add_large_page_to_freelist(head,
 673                                                    (void *)((char *)head + off),
 674                                                    sp_bits);
 675                 }
 676         } else {
 677                 for (off = ha->off;
 678                      off < ha->off + ha->len;
 679                      off += 1UL << sp_bits) {
 680                         add_small_page_to_freelist(head,
 681                                                    (void *)((char *)head + off),
 682                                                    sp_bits);
 683                 }
 684         }
 685         del_from_huge(head, ha);
 686         alloc_free(head, poolsize, ha);
 687 }
 688
 689 static COLD unsigned long huge_size(struct header *head, void *p)
 690 {
 691         unsigned long i, off = (char *)p - (char *)head;
 692         struct huge_alloc *ha;
 693
 694         for (i = head->huge; i; i = ha->next) {
 695                 ha = (void *)((char *)head + i);
 696                 if (off == ha->off) {
 697                         return ha->len;
 698                 }
 699         }
 700         abort();
 701 }
 702
 703 void *alloc_get(void *pool, unsigned long poolsize,
 704                 unsigned long size, unsigned long align)
 705 {
 706         struct header *head = pool;
 707         unsigned int bucket;
 708         unsigned long i;
 709         struct bucket_state *bs;
 710         struct page_header *ph;
 711         unsigned int sp_bits;
 712
 713         if (poolsize < MIN_USEFUL_SIZE) {
 714                 return tiny_alloc_get(pool, poolsize, size, align);
 715         }
 716
 717         size = align_up(size, align);
 718         if (unlikely(!size))
 719                 size = 1;
 720         bucket = size_to_bucket(size);
 721
 722         sp_bits = small_page_bits(poolsize);
 723
 724         if (bucket >= max_bucket(sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE)) {
 725                 return huge_alloc(pool, poolsize, size, align);
 726         }
 727
 728         bs = &head->bs[bucket];
 729
 730         if (!bs->page_list) {
 731                 if (large_page_bucket(bucket, sp_bits))
 732                         bs->page_list = get_large_page(head, poolsize,
 733                                                        sp_bits);
 734                 else
 735                         bs->page_list = get_small_page(head, poolsize,
 736                                                        sp_bits);
 737                 /* FIXME: Try large-aligned alloc?  Header stuffing? */
 738                 if (unlikely(!bs->page_list))
 739                         return NULL;
 740                 ph = from_pgnum(head, bs->page_list, sp_bits);
 741                 ph->bucket = bucket;
 742                 ph->elements_used = 0;
 743                 ph->next = 0;
 744                 memset(ph->used, 0, used_size(bs->elements_per_page));
 745         }
 746
 747         ph = from_pgnum(head, bs->page_list, sp_bits);
 748
 749         i = find_free_bit(ph->used);
 750         set_bit(ph->used, i);
 751         ph->elements_used++;
 752
 753         /* check if this page is now full */
 754         if (unlikely(ph->elements_used == bs->elements_per_page)) {
 755                 del_from_bucket_list(head, bs, ph, sp_bits);
 756                 add_to_bucket_full_list(head, bs, ph, sp_bits);
 757         }
 758
 759         return (char *)ph + page_header_size(ph->bucket / INTER_BUCKET_SPACE,
 760                                              bs->elements_per_page)
 761                + i * bucket_to_size(bucket);
 762 }
 763
 764 void alloc_free(void *pool, unsigned long poolsize, void *free)
 765 {
 766         struct header *head = pool;
 767         struct bucket_state *bs;
 768         unsigned int sp_bits;
 769         unsigned long i, pgnum, pgoffset, offset = (char *)free - (char *)pool;
 770         bool smallpage;
 771         struct page_header *ph;
 772
 773         if (poolsize < MIN_USEFUL_SIZE) {
 774                 tiny_alloc_free(pool, poolsize, free);
 775                 return;
 776         }
 777
 778         /* Get page header. */
 779         sp_bits = small_page_bits(poolsize);
 780         pgnum = offset >> sp_bits;
 781
 782         /* Big page? Round down further. */
 783         if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
 784                 smallpage = false;
 785                 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
 786         } else
 787                 smallpage = true;
 788
 789         /* Step back to page header. */
 790         ph = from_pgnum(head, pgnum, sp_bits);
 791         if ((void *)ph == free) {
 792                 huge_free(head, poolsize, free);
 793                 return;
 794         }
 795
 796         bs = &head->bs[ph->bucket];
 797         pgoffset = offset - (pgnum << sp_bits)
 798                 - page_header_size(ph->bucket / INTER_BUCKET_SPACE,
 799                                    bs->elements_per_page);
 800
 801         if (unlikely(ph->elements_used == bs->elements_per_page)) {
 802                 del_from_bucket_full_list(head, bs, ph, sp_bits);
 803                 add_to_bucket_list(head, bs, ph, sp_bits);
 804         }
 805
 806         /* Which element are we? */
 807         i = pgoffset / bucket_to_size(ph->bucket);
 808         clear_bit(ph->used, i);
 809         ph->elements_used--;
 810
 811         if (unlikely(ph->elements_used == 0)) {
 812                 bs = &head->bs[ph->bucket];
 813                 del_from_bucket_list(head, bs, ph, sp_bits);
 814                 if (smallpage)
 815                         add_small_page_to_freelist(head, ph, sp_bits);
 816                 else
 817                         add_large_page_to_freelist(head, ph, sp_bits);
 818         }
 819 }
 820
 821 unsigned long alloc_size(void *pool, unsigned long poolsize, void *p)
 822 {
 823         struct header *head = pool;
 824         unsigned int pgnum, sp_bits;
 825         unsigned long offset = (char *)p - (char *)pool;
 826         struct page_header *ph;
 827
 828         if (poolsize < MIN_USEFUL_SIZE)
 829                 return tiny_alloc_size(pool, poolsize, p);
 830
 831         /* Get page header. */
 832         sp_bits = small_page_bits(poolsize);
 833         pgnum = offset >> sp_bits;
 834
 835         /* Big page? Round down further. */
 836         if (test_bit(head->pagesize, pgnum >> BITS_FROM_SMALL_TO_LARGE_PAGE))
 837                 pgnum &= ~(SMALL_PAGES_PER_LARGE_PAGE - 1);
 838
 839         /* Step back to page header. */
 840         ph = from_pgnum(head, pgnum, sp_bits);
 841         if ((void *)ph == p)
 842                 return huge_size(head, p);
 843
 844         return bucket_to_size(ph->bucket);
 845 }
 846
 847 /* Useful for gdb breakpoints. */
 848 static bool check_fail(void)
 849 {
 850         return false;
 851 }
 852
 853 static unsigned long count_bits(const unsigned long bitmap[],
 854                                 unsigned long limit)
 855 {
 856         unsigned long i, count = 0;
 857
 858         while (limit >= BITS_PER_LONG) {
 859                 count += popcount(bitmap[0]);
 860                 bitmap++;
 861                 limit -= BITS_PER_LONG;
 862         }
 863
 864         for (i = 0; i < limit; i++)
 865                 if (test_bit(bitmap, i))
 866                         count++;
 867         return count;
 868 }
 869
 870 static bool out_of_bounds(unsigned long pgnum,
 871                           unsigned int sp_bits,
 872                           unsigned long pagesize,
 873                           unsigned long poolsize)
 874 {
 875         if (((pgnum << sp_bits) >> sp_bits) != pgnum)
 876                 return true;
 877
 878         if ((pgnum << sp_bits) > poolsize)
 879                 return true;
 880
 881         return ((pgnum << sp_bits) + pagesize > poolsize);
 882 }
 883
 884 static bool check_bucket(struct header *head,
 885                          unsigned long poolsize,
 886                          unsigned long pages[],
 887                          struct bucket_state *bs,
 888                          unsigned int bindex)
 889 {
 890         bool lp_bucket;
 891         struct page_header *ph;
 892         unsigned long taken, i, prev, pagesize, sp_bits, lp_bits;
 893
 894         sp_bits = small_page_bits(poolsize);
 895         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 896
 897         lp_bucket = large_page_bucket(bindex, sp_bits);
 898
 899         pagesize = 1UL << (lp_bucket ? lp_bits : sp_bits);
 900
 901         /* This many elements fit? */
 902         taken = page_header_size(bindex / INTER_BUCKET_SPACE,
 903                                  bs->elements_per_page);
 904         taken += bucket_to_size(bindex) * bs->elements_per_page;
 905         if (taken > pagesize)
 906                 return check_fail();
 907
 908         /* One more wouldn't fit? */
 909         taken = page_header_size(bindex / INTER_BUCKET_SPACE,
 910                                  bs->elements_per_page + 1);
 911         taken += bucket_to_size(bindex) * (bs->elements_per_page + 1);
 912         if (taken <= pagesize)
 913                 return check_fail();
 914
 915         /* Walk used list. */
 916         prev = 0;
 917         for (i = bs->page_list; i; i = ph->next) {
 918                 /* Bad pointer? */
 919                 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
 920                         return check_fail();
 921                 /* Wrong size page? */
 922                 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
 923                     != lp_bucket)
 924                         return check_fail();
 925                 /* Large page not on boundary? */
 926                 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
 927                         return check_fail();
 928                 ph = from_pgnum(head, i, sp_bits);
 929                 /* Linked list corrupt? */
 930                 if (ph->prev != prev)
 931                         return check_fail();
 932                 /* Already seen this page? */
 933                 if (test_bit(pages, i))
 934                         return check_fail();
 935                 set_bit(pages, i);
 936                 /* Empty or full? */
 937                 if (ph->elements_used == 0)
 938                         return check_fail();
 939                 if (ph->elements_used >= bs->elements_per_page)
 940                         return check_fail();
 941                 /* Used bits don't agree? */
 942                 if (ph->elements_used != count_bits(ph->used,
 943                                                     bs->elements_per_page))
 944                         return check_fail();
 945                 /* Wrong bucket? */
 946                 if (ph->bucket != bindex)
 947                         return check_fail();
 948                 prev = i;
 949         }
 950
 951         /* Walk full list. */
 952         prev = 0;
 953         for (i = bs->full_list; i; i = ph->next) {
 954                 /* Bad pointer? */
 955                 if (out_of_bounds(i, sp_bits, pagesize, poolsize))
 956                         return check_fail();
 957                 /* Wrong size page? */
 958                 if (!!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE)
 959                     != lp_bucket)
 960                 /* Large page not on boundary? */
 961                 if (lp_bucket && (i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
 962                         return check_fail();
 963                 ph = from_pgnum(head, i, sp_bits);
 964                 /* Linked list corrupt? */
 965                 if (ph->prev != prev)
 966                         return check_fail();
 967                 /* Already seen this page? */
 968                 if (test_bit(pages, i))
 969                         return check_fail();
 970                 set_bit(pages, i);
 971                 /* Not full? */
 972                 if (ph->elements_used != bs->elements_per_page)
 973                         return check_fail();
 974                 /* Used bits don't agree? */
 975                 if (ph->elements_used != count_bits(ph->used,
 976                                                     bs->elements_per_page))
 977                         return check_fail();
 978                 /* Wrong bucket? */
 979                 if (ph->bucket != bindex)
 980                         return check_fail();
 981                 prev = i;
 982         }
 983         return true;
 984 }
 985
 986 bool alloc_check(void *pool, unsigned long poolsize)
 987 {
 988         struct header *head = pool;
 989         unsigned long prev, i, lp_bits, sp_bits, header_size, num_buckets;
 990         struct page_header *ph;
 991         struct huge_alloc *ha;
 992         unsigned long pages[MAX_SMALL_PAGES / BITS_PER_LONG] = { 0 };
 993
 994         if (poolsize < MIN_USEFUL_SIZE)
 995                 return tiny_alloc_check(pool, poolsize);
 996
 997         sp_bits = small_page_bits(poolsize);
 998         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
 999
1000         num_buckets = max_bucket(lp_bits);
1001
1002         header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1003
1004         /* First, set all bits taken by header. */
1005         for (i = 0; i < header_size; i += (1UL << sp_bits))
1006                 set_bit(pages, i >> sp_bits);
1007
1008         /* Check small page free list. */
1009         prev = 0;
1010         for (i = head->small_free_list; i; i = ph->next) {
1011                 /* Bad pointer? */
1012                 if (out_of_bounds(i, sp_bits, 1UL << sp_bits, poolsize))
1013                         return check_fail();
1014                 /* Large page? */
1015                 if (test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1016                         return check_fail();
1017                 ph = from_pgnum(head, i, sp_bits);
1018                 /* Linked list corrupt? */
1019                 if (ph->prev != prev)
1020                         return check_fail();
1021                 /* Already seen this page? */
1022                 if (test_bit(pages, i))
1023                         return check_fail();
1024                 set_bit(pages, i);
1025                 prev = i;
1026         }
1027
1028         /* Check large page free list. */
1029         prev = 0;
1030         for (i = head->large_free_list; i; i = ph->next) {
1031                 /* Bad pointer? */
1032                 if (out_of_bounds(i, sp_bits, 1UL << lp_bits, poolsize))
1033                         return check_fail();
1034                 /* Not large page? */
1035                 if (!test_bit(head->pagesize, i >> BITS_FROM_SMALL_TO_LARGE_PAGE))
1036                         return check_fail();
1037                 /* Not page boundary? */
1038                 if ((i % SMALL_PAGES_PER_LARGE_PAGE) != 0)
1039                         return check_fail();
1040                 ph = from_pgnum(head, i, sp_bits);
1041                 /* Linked list corrupt? */
1042                 if (ph->prev != prev)
1043                         return check_fail();
1044                 /* Already seen this page? */
1045                 if (test_bit(pages, i))
1046                         return check_fail();
1047                 set_bit(pages, i);
1048                 prev = i;
1049         }
1050
1051         /* Check the buckets. */
1052         for (i = 0; i < max_bucket(lp_bits); i++) {
1053                 struct bucket_state *bs = &head->bs[i];
1054
1055                 if (!check_bucket(head, poolsize, pages, bs, i))
1056                         return false;
1057         }
1058
1059         /* Check the huge alloc list. */
1060         prev = 0;
1061         for (i = head->huge; i; i = ha->next) {
1062                 unsigned long pgbits, j;
1063
1064                 /* Bad pointer? */
1065                 if (i >= poolsize || i + sizeof(*ha) > poolsize)
1066                         return check_fail();
1067                 ha = (void *)((char *)head + i);
1068
1069                 /* Check contents of ha. */
1070                 if (ha->off > poolsize || ha->off + ha->len > poolsize)
1071                         return check_fail();
1072
1073                 /* Large or small page? */
1074                 pgbits = test_bit(head->pagesize, ha->off >> lp_bits)
1075                         ? lp_bits : sp_bits;
1076
1077                 /* Not page boundary? */
1078                 if ((ha->off % (1UL << pgbits)) != 0)
1079                         return check_fail();
1080
1081                 /* Not page length? */
1082                 if ((ha->len % (1UL << pgbits)) != 0)
1083                         return check_fail();
1084
1085                 /* Linked list corrupt? */
1086                 if (ha->prev != prev)
1087                         return check_fail();
1088
1089                 for (j = ha->off; j < ha->off + ha->len; j += (1UL<<sp_bits)) {
1090                         /* Already seen this page? */
1091                         if (test_bit(pages, j >> sp_bits))
1092                                 return check_fail();
1093                         set_bit(pages, j >> sp_bits);
1094                 }
1095
1096                 prev = i;
1097         }
1098
1099         /* Make sure every page accounted for. */
1100         for (i = 0; i < poolsize >> sp_bits; i++) {
1101                 if (!test_bit(pages, i))
1102                         return check_fail();
1103                 if (test_bit(head->pagesize,
1104                              i >> BITS_FROM_SMALL_TO_LARGE_PAGE)) {
1105                         /* Large page, skip rest. */
1106                         i += SMALL_PAGES_PER_LARGE_PAGE - 1;
1107                 }
1108         }
1109
1110         return true;
1111 }
1112
1113 static unsigned long print_overhead(FILE *out, const char *desc,
1114                                     unsigned long bytes,
1115                                     unsigned long poolsize)
1116 {
1117         fprintf(out, "Overhead (%s): %lu bytes (%.3g%%)\n",
1118                 desc, bytes, 100.0 * bytes / poolsize);
1119         return bytes;
1120 }
1121
1122 static unsigned long count_list(struct header *head,
1123                                 u16 pgnum,
1124                                 unsigned int sp_bits,
1125                                 unsigned long *total_elems)
1126 {
1127         struct page_header *p;
1128         unsigned long ret = 0;
1129
1130         while (pgnum) {
1131                 p = from_pgnum(head, pgnum, sp_bits);
1132                 if (total_elems)
1133                         (*total_elems) += p->elements_used;
1134                 ret++;
1135                 pgnum = p->next;
1136         }
1137         return ret;
1138 }
1139
1140 static unsigned long visualize_bucket(FILE *out, struct header *head,
1141                                       unsigned int bucket,
1142                                       unsigned long poolsize,
1143                                       unsigned int sp_bits)
1144 {
1145         unsigned long num_full, num_partial, num_pages, page_size,
1146                 elems, hdr_min, hdr_size, elems_per_page, overhead = 0;
1147
1148         elems_per_page = head->bs[bucket].elements_per_page;
1149
1150         /* If we used byte-based bitmaps, we could get pg hdr to: */
1151         hdr_min = sizeof(struct page_header)
1152                 - sizeof(((struct page_header *)0)->used)
1153                 + align_up(elems_per_page, CHAR_BIT) / CHAR_BIT;
1154         hdr_size = page_header_size(bucket / INTER_BUCKET_SPACE,
1155                                     elems_per_page);
1156
1157         elems = 0;
1158         num_full = count_list(head, head->bs[bucket].full_list, sp_bits,
1159                               &elems);
1160         num_partial = count_list(head, head->bs[bucket].page_list, sp_bits,
1161                                  &elems);
1162         num_pages = num_full + num_partial;
1163         if (!num_pages)
1164                 return 0;
1165
1166         fprintf(out, "Bucket %u (%lu bytes):"
1167                 " %lu full, %lu partial = %lu elements\n",
1168                 bucket, bucket_to_size(bucket), num_full, num_partial, elems);
1169         /* Strict requirement of page header size. */
1170         overhead += print_overhead(out, "page headers",
1171                                    hdr_min * num_pages, poolsize);
1172         /* Gap between minimal page header and actual start. */
1173         overhead += print_overhead(out, "page post-header alignments",
1174                                    (hdr_size - hdr_min) * num_pages, poolsize);
1175         /* Between last element and end of page. */
1176         page_size = (1UL << sp_bits);
1177         if (large_page_bucket(bucket, sp_bits))
1178                 page_size <<= BITS_FROM_SMALL_TO_LARGE_PAGE;
1179
1180         overhead += print_overhead(out, "page tails",
1181                                    (page_size - (hdr_size
1182                                                  + (elems_per_page
1183                                                     * bucket_to_size(bucket))))
1184                                    * num_pages, poolsize);
1185         return overhead;
1186 }
1187
1188 void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
1189 {
1190         struct header *head = pool;
1191         unsigned long i, lp_bits, sp_bits, header_size, num_buckets, count,
1192                 overhead = 0;
1193
1194         fprintf(out, "Pool %p size %lu: (%s allocator)\n", pool, poolsize,
1195                 poolsize < MIN_USEFUL_SIZE ? "tiny" : "standard");
1196
1197         if (poolsize < MIN_USEFUL_SIZE) {
1198                 tiny_alloc_visualize(out, pool, poolsize);
1199                 return;
1200         }
1201
1202         sp_bits = small_page_bits(poolsize);
1203         lp_bits = sp_bits + BITS_FROM_SMALL_TO_LARGE_PAGE;
1204
1205         num_buckets = max_bucket(lp_bits);
1206         header_size = sizeof(*head) + sizeof(head->bs) * (num_buckets-1);
1207
1208         fprintf(out, "Large page size %lu, small page size %lu.\n",
1209                 1UL << lp_bits, 1UL << sp_bits);
1210         overhead += print_overhead(out, "unused pool tail",
1211                                    poolsize % (1UL << lp_bits), poolsize);
1212         fprintf(out, "Main header %lu bytes (%lu small pages).\n",
1213                 header_size, align_up(header_size, 1UL << sp_bits) >> sp_bits);
1214         overhead += print_overhead(out, "partial header page",
1215                                    align_up(header_size, 1UL << sp_bits)
1216                                    - header_size, poolsize);
1217         /* Total large pages. */
1218         i = count_bits(head->pagesize, poolsize >> lp_bits);
1219         /* Used pages. */
1220         count = i - count_list(head, head->large_free_list, sp_bits, NULL);
1221         fprintf(out, "%lu/%lu large pages used (%.3g%%)\n",
1222                 count, i, count ? 100.0 * count / i : 0.0);
1223
1224         /* Total small pages. */
1225         i = ((poolsize >> lp_bits) - i) << BITS_FROM_SMALL_TO_LARGE_PAGE;
1226         /* Used pages */
1227         count = i - count_list(head, head->small_free_list, sp_bits, NULL);
1228         fprintf(out, "%lu/%lu small pages used (%.3g%%)\n",
1229                 count, i, count ? 100.0 * count / i : 0.0);
1230
1231         /* Summary of each bucket. */
1232         fprintf(out, "%lu buckets:\n", num_buckets);
1233         for (i = 0; i < num_buckets; i++)
1234                 overhead += visualize_bucket(out, head, i, poolsize, sp_bits);
1235
1236         print_overhead(out, "total", overhead, poolsize);
1237 }