X-Git-Url: http://git.ozlabs.org/?p=ccan;a=blobdiff_plain;f=alloc%2Falloc.c;h=c209a70f39d39cc09e850c01363e4fb8cbbea015;hp=48bc2e5e21aaaeb5337f5a7c5734e8291e1b0598;hb=f033b4f38211e299232e226d5c39ef0e7f73475c;hpb=b3d0388fba47907e85e6db7c8c0e1d18ed759994

diff --git a/alloc/alloc.c b/alloc/alloc.c
index 48bc2e5e..c209a70f 100644
--- a/alloc/alloc.c
+++ b/alloc/alloc.c
@@ -42,25 +42,58 @@ struct metaheader
 	/* Bits start here. */
 };
 
-#define BITS_PER_PAGE 2
-/* FIXME: Don't use page states for bitblock.  It's tacky and confusing. */
-enum page_state
+/* Assumes a is a power of two. */
+static unsigned long align_up(unsigned long x, unsigned long a)
+{
+	return (x + a - 1) & ~(a - 1);
+}
+
+static unsigned long div_up(unsigned long x, unsigned long a)
+{
+	return (x + a - 1) / a;
+}
+
+/* It turns out that we spend a lot of time dealing with bit pairs.
+ * These routines manipulate them.
+ */
+static uint8_t get_bit_pair(const uint8_t *bits, unsigned long index)
+{
+	return bits[index * 2 / CHAR_BIT] >> (index * 2 % CHAR_BIT) & 3;
+}
+
+static void set_bit_pair(uint8_t *bits, unsigned long index, uint8_t val)
+{
+	bits[index * 2 / CHAR_BIT] &= ~(3 << (index * 2 % CHAR_BIT));
+	bits[index * 2 / CHAR_BIT] |= (val << (index * 2 % CHAR_BIT));
+}
+
+/* This is used for page states and subpage allocations */
+enum alloc_state
 {
 	FREE,
 	TAKEN,
 	TAKEN_START,
-	SUBPAGE,
+	SPECIAL,	/* Sub-page allocation for page states. */
 };
 
-/* Assumes a is a power of two. */
-static unsigned long align_up(unsigned long x, unsigned long a)
+/* The types for subpage metadata. */
+enum sub_metadata_type
 {
-	return (x + a - 1) & ~(a - 1);
+	/* FREE is same as alloc state */
+	BITMAP = 1,
+};
+
+/* Page states are represented by bitpairs, at the start of the pool. */
+#define BITS_PER_PAGE 2
+
+static enum alloc_state get_page_state(const void *pool, unsigned long page)
+{
+	return get_bit_pair(pool, page);
 }
 
-static unsigned long div_up(unsigned long x, unsigned long a)
+static void set_page_state(void *pool, unsigned long page, enum alloc_state s)
 {
-	return (x + a - 1) / a;
+	set_bit_pair(pool, page, s);
 }
 
 /* The offset of metadata for a subpage allocation is found at the end
@@ -69,24 +102,13 @@ static unsigned long div_up(unsigned long x, unsigned long a)
 
 /* This is the length of metadata in bits.  It consists of two bits
  * for every BITMAP_GRANULARITY of usable bytes in the page, then two
- * bits for the TAKEN tailer.. */
+ * bits for the tailer.. */
 #define BITMAP_METABITLEN						\
     ((div_up(SUBPAGE_METAOFF, BITMAP_GRANULARITY) + 1) * BITS_PER_PAGE)
 
 /* This is the length in bytes. */
 #define BITMAP_METALEN (div_up(BITMAP_METABITLEN, CHAR_BIT))
 
-static enum page_state get_page_state(const uint8_t *bits, unsigned long page)
-{
-	return bits[page * 2 / CHAR_BIT] >> (page * 2 % CHAR_BIT) & 3;
-}
-
-static void set_page_state(uint8_t *bits, unsigned long page, enum page_state s)
-{
-	bits[page * 2 / CHAR_BIT] &= ~(3 << (page * 2 % CHAR_BIT));
-	bits[page * 2 / CHAR_BIT] |= ((uint8_t)s << (page * 2 % CHAR_BIT));
-}
-
 static struct metaheader *first_mheader(void *pool, unsigned long poolsize)
 {
 	unsigned int pagestatelen;
@@ -121,14 +143,14 @@ void alloc_init(void *pool, unsigned long poolsize)
 
 	mh = first_mheader(pool, poolsize);
 
-	/* len covers all page states, plus the metaheader. */
-	len = (char *)(mh + 1) - (char *)pool;
-	/* Mark all page states FREE */
+	/* Mark all page states FREE, and all of metaheader bitmap which takes
+	 * rest of first page. */
+	len = align_up(pool_offset(pool, mh + 1), getpagesize());
 	BUILD_ASSERT(FREE == 0);
 	memset(pool, 0, len);
 
-	/* metaheader len takes us up to next page boundary. */
-	mh->metalen = align_up(len, getpagesize()) - len;
+	/* Set up metalen */
+	mh->metalen = len - pool_offset(pool, mh + 1);
 
 	/* Mark the pagestate and metadata page(s) allocated. */
 	set_page_state(pool, 0, TAKEN_START);
@@ -136,9 +158,12 @@ void alloc_init(void *pool, unsigned long poolsize)
 		set_page_state(pool, i, TAKEN);
 }
 
-/* Two bits per element, representing page states.  Returns -1 on fail. */
-static long alloc_from_bitmap(uint8_t *bits, unsigned long elems,
-			      unsigned long want, unsigned long align)
+/* Two bits per element, representing page states.  Returns 0 on fail.
+ * off is used to allocate from subpage bitmaps, which use the first 2
+ * bits as the type, so the real bitmap is offset by 1. */
+static unsigned long alloc_from_bitmap(uint8_t *bits, unsigned long off,
+				       unsigned long elems,
+				       unsigned long want, unsigned long align)
 {
 	long i;
 	unsigned long free;
@@ -146,7 +171,7 @@ static long alloc_from_bitmap(uint8_t *bits, unsigned long elems,
 	free = 0;
 	/* We allocate from far end, to increase ability to expand metadata. */
 	for (i = elems - 1; i >= 0; i--) {
-		switch (get_page_state(bits, i)) {
+		switch (get_bit_pair(bits, off+i)) {
 		case FREE:
 			if (++free >= want) {
 				unsigned long j;
@@ -155,13 +180,13 @@ static long alloc_from_bitmap(uint8_t *bits, unsigned long elems,
 				if (align && i % align)
 					continue;
 
+				set_bit_pair(bits, off+i, TAKEN_START);
 				for (j = i+1; j < i + want; j++)
-					set_page_state(bits, j, TAKEN);
-				set_page_state(bits, i, TAKEN_START);
-				return i;
+					set_bit_pair(bits, off+j, TAKEN);
+				return off+i;
 			}
 			break;
-		case SUBPAGE:
+		case SPECIAL:
 		case TAKEN_START:
 		case TAKEN:
 			free = 0;
@@ -169,44 +194,14 @@ static long alloc_from_bitmap(uint8_t *bits, unsigned long elems,
 		}
 	}
 
-	return -1;
+	return 0;
 }
 
 static unsigned long alloc_get_pages(void *pool, unsigned long poolsize,
 				     unsigned long pages, unsigned long align)
 {
-	long i;
-	unsigned long free;
-
-	free = 0;
-	/* We allocate from far end, to increase ability to expand metadata. */
-	for (i = poolsize / getpagesize() - 1; i >= 0; i--) {
-		switch (get_page_state(pool, i)) {
-		case FREE:
-			if (++free >= pages) {
-				unsigned long j, addr;
-
-				addr = (unsigned long)pool + i * getpagesize();
-
-				/* They might ask for multi-page alignment. */
-				if (addr % align)
-					continue;
-
-				for (j = i+1; j < i + pages; j++)
-					set_page_state(pool, j, TAKEN);
-				set_page_state(pool, i, TAKEN_START);
-				return i;
-			}
-			break;
-		case SUBPAGE:
-		case TAKEN_START:
-		case TAKEN:
-			free = 0;
-			break;
-		}
-	}
-
-	return 0;
+	return alloc_from_bitmap(pool, 0, poolsize / getpagesize(), pages,
+				 align / getpagesize());
 }
 
 /* Offset to metadata is at end of page. */
@@ -230,43 +225,46 @@ static unsigned long sub_page_alloc(void *pool, unsigned long page,
 				    unsigned long size, unsigned long align)
 {
 	uint8_t *bits = get_page_metadata(pool, page);
-	long i;
+	unsigned long i;
 
-	/* TAKEN at end means a bitwise alloc. */
-	assert(get_page_state(bits, getpagesize()/BITMAP_GRANULARITY - 1)
-	       == TAKEN);
+	/* TAKEN at start means a bitwise alloc. */
+	assert(get_bit_pair(bits, 0) == BITMAP);
 
-	/* Our bits are the same as the page bits. */
-	i = alloc_from_bitmap(bits, SUBPAGE_METAOFF/BITMAP_GRANULARITY,
+	/* We use a standart bitmap, but offset because of that BITMAP
+	 * header. */
+	i = alloc_from_bitmap(bits, 1, SUBPAGE_METAOFF/BITMAP_GRANULARITY,
 			      div_up(size, BITMAP_GRANULARITY),
 			      align / BITMAP_GRANULARITY);
 
 	/* Can't allocate? */
-	if (i < 0)
+	if (i == 0)
 		return 0;
 
-	return page*getpagesize() + i*BITMAP_GRANULARITY;
+	/* i-1 because of the header. */
+	return page*getpagesize() + (i-1)*BITMAP_GRANULARITY;
 }
 
-static uint8_t *alloc_metaspace(struct metaheader *mh, unsigned long bytes)
+static uint8_t *alloc_metaspace(struct metaheader *mh, unsigned long bytes,
+				enum sub_metadata_type type)
 {
 	uint8_t *meta = (uint8_t *)(mh + 1);
 	unsigned long free = 0, len;
-	long i;
+	unsigned long i;
 
 	/* TAKEN tags end a subpage alloc. */
-	for (i = mh->metalen * CHAR_BIT / BITS_PER_PAGE - 1; i >= 0; i -= len) {
-		switch (get_page_state(meta, i)) {
+	for (i = 0; i < mh->metalen * CHAR_BIT / BITS_PER_PAGE; i += len) {
+		switch (get_bit_pair(meta, i)) {
 		case FREE:
 			len = 1;
 			free++;
 			if (free == bytes * CHAR_BIT / BITS_PER_PAGE) {
-				/* TAKEN marks end of metablock. */
-				set_page_state(meta, i + free - 1, TAKEN);
-				return meta + i / (CHAR_BIT / BITS_PER_PAGE);
+				/* Mark this as a bitmap. */
+				set_bit_pair(meta, i - free + 1, type);
+				return meta + (i - free + 1)
+					/ (CHAR_BIT / BITS_PER_PAGE);
 			}
 			break;
-		case TAKEN:
+		case BITMAP:
 			/* Skip over this allocated part. */
 			len = BITMAP_METALEN * CHAR_BIT / BITS_PER_PAGE;
 			free = 0;
@@ -281,13 +279,13 @@ static uint8_t *alloc_metaspace(struct metaheader *mh, unsigned long bytes)
 
 /* We need this many bytes of metadata. */
 static uint8_t *new_metadata(void *pool, unsigned long poolsize,
-			     unsigned long bytes)
+			     unsigned long bytes, enum sub_metadata_type type)
 {
 	struct metaheader *mh, *newmh;
 	unsigned long page;
 
 	for (mh = first_mheader(pool,poolsize); mh; mh = next_mheader(pool,mh)){
-		uint8_t *meta = alloc_metaspace(mh, bytes);
+		uint8_t *meta = alloc_metaspace(mh, bytes, type);
 
 		if (meta)
 			return meta;
@@ -310,7 +308,7 @@ static uint8_t *new_metadata(void *pool, unsigned long poolsize,
 		BUILD_ASSERT(FREE == 0);
 		memset((char *)pool + nextpage, 0, getpagesize());
 		mh->metalen += getpagesize();
-		return alloc_metaspace(mh, bytes);
+		return alloc_metaspace(mh, bytes, type);
 	}
 
 	/* No metadata left at all? */
@@ -328,7 +326,7 @@ static uint8_t *new_metadata(void *pool, unsigned long poolsize,
 	newmh->next = mh->next;
 	mh->next = pool_offset(pool, newmh);
 
-	return alloc_metaspace(newmh, bytes);
+	return alloc_metaspace(newmh, bytes, type);
 }
 
 static void alloc_free_pages(void *pool, unsigned long pagenum)
@@ -348,7 +346,7 @@ static unsigned long alloc_sub_page(void *pool, unsigned long poolsize,
 	/* Look for partial page. */
 	for (i = 0; i < poolsize / getpagesize(); i++) {
 		unsigned long ret;
-		if (get_page_state(pool, i) != SUBPAGE)
+		if (get_page_state(pool, i) != SPECIAL)
 			continue;
 
 		ret = sub_page_alloc(pool, i, size, align);
@@ -362,14 +360,14 @@ static unsigned long alloc_sub_page(void *pool, unsigned long poolsize,
 		return 0;
 
 	/* Get metadata for page. */
-	metadata = new_metadata(pool, poolsize, BITMAP_METALEN);
+	metadata = new_metadata(pool, poolsize, BITMAP_METALEN, BITMAP);
 	if (!metadata) {
 		alloc_free_pages(pool, i);
 		return 0;
 	}
 
-	/* Actually, this is a SUBPAGE page now. */
-	set_page_state(pool, i, SUBPAGE);
+	/* Actually, this is a subpage page now. */
+	set_page_state(pool, i, SPECIAL);
 
 	/* Set metadata pointer for page. */
 	set_page_metadata(pool, i, metadata);
@@ -387,16 +385,17 @@ static bool clean_empty_subpages(void *pool, unsigned long poolsize)
 	for (i = 0; i < poolsize/getpagesize(); i++) {
 		uint8_t *meta;
 		unsigned int j;
-		if (get_page_state(pool, i) != SUBPAGE)
+		if (get_page_state(pool, i) != SPECIAL)
 			continue;
 
 		meta = get_page_metadata(pool, i);
-		for (j = 0; j < SUBPAGE_METAOFF/BITMAP_GRANULARITY; j++)
-			if (get_page_state(meta, j) != FREE)
+		/* Skip the header (first bit of metadata). */
+		for (j = 1; j < SUBPAGE_METAOFF/BITMAP_GRANULARITY+1; j++)
+			if (get_bit_pair(meta, j) != FREE)
 				break;
 
 		/* So, is this page totally empty? */
-		if (j == SUBPAGE_METAOFF/BITMAP_GRANULARITY) {
+		if (j == SUBPAGE_METAOFF/BITMAP_GRANULARITY+1) {
 			set_page_state(pool, i, FREE);
 			progress = true;
 		}
@@ -497,10 +496,13 @@ static void subpage_free(void *pool, unsigned long pagenum, void *free)
 
 	off /= BITMAP_GRANULARITY;
 
-	set_page_state(metadata, off++, FREE);
+	/* Offset by one because first bit is used for header. */
+	off++;
+
+	set_bit_pair(metadata, off++, FREE);
 	while (off < SUBPAGE_METAOFF / BITMAP_GRANULARITY
-	       && get_page_state(metadata, off) == TAKEN)
-		set_page_state(metadata, off++, FREE);
+	       && get_bit_pair(metadata, off) == TAKEN)
+		set_bit_pair(metadata, off++, FREE);
 }
 
 void alloc_free(void *pool, unsigned long poolsize, void *free)
@@ -519,7 +521,7 @@ void alloc_free(void *pool, unsigned long poolsize, void *free)
 
 	pagenum = pool_offset(pool, free) / getpagesize();
 
-	if (get_page_state(pool, pagenum) == SUBPAGE)
+	if (get_page_state(pool, pagenum) == SPECIAL)
 		subpage_free(pool, pagenum, free);
 	else {
 		assert((unsigned long)free % getpagesize() == 0);
@@ -548,7 +550,7 @@ static bool check_subpage(void *pool, unsigned long poolsize,
 {
 	unsigned long *mhoff = metadata_off(pool, page);
 	unsigned int i;
-	enum page_state last_state = FREE;
+	enum alloc_state last_state = FREE;
 
 	if (*mhoff + sizeof(struct metaheader) > poolsize)
 		return false;
@@ -560,17 +562,17 @@ static bool check_subpage(void *pool, unsigned long poolsize,
 	if (!is_metadata_page(pool, poolsize, *mhoff / getpagesize()))
 		return false;
 
-	/* Marker at end of subpage allocation is "taken" */
-	if (get_page_state((uint8_t *)pool + *mhoff,
-			   getpagesize()/BITMAP_GRANULARITY - 1) != TAKEN)
+	/* Header at start of subpage allocation */
+	if (get_bit_pair((uint8_t *)pool + *mhoff, 0) != BITMAP)
 		return false;
 
 	for (i = 0; i < SUBPAGE_METAOFF / BITMAP_GRANULARITY; i++) {
-		enum page_state state;
+		enum alloc_state state;
 
-		state = get_page_state((uint8_t *)pool + *mhoff, i);
+		/* +1 because header is the first byte. */
+		state = get_bit_pair((uint8_t *)pool + *mhoff, i+1);
 		switch (state) {
-		case SUBPAGE:
+		case SPECIAL:
 			return false;
 		case TAKEN:
 			if (last_state == FREE)
@@ -588,7 +590,7 @@ bool alloc_check(void *pool, unsigned long poolsize)
 {
 	unsigned long i;
 	struct metaheader *mh;
-	enum page_state last_state = FREE;
+	enum alloc_state last_state = FREE;
 	bool was_metadata = false;
 
 	if (poolsize < MIN_SIZE)
@@ -621,7 +623,7 @@ bool alloc_check(void *pool, unsigned long poolsize)
 	}
 
 	for (i = 0; i < poolsize / getpagesize(); i++) {
-		enum page_state state = get_page_state(pool, i);
+		enum alloc_state state = get_page_state(pool, i);
 		bool is_metadata = is_metadata_page(pool, poolsize,i);
 
 		switch (state) {
@@ -638,7 +640,7 @@ bool alloc_check(void *pool, unsigned long poolsize)
 			if (is_metadata != was_metadata)
 				return false;
 			break;
-		case SUBPAGE:
+		case SPECIAL:
 			/* Check metadata pointer etc. */
 			if (!check_subpage(pool, poolsize, i))
 				return false;
@@ -679,16 +681,15 @@ void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
 
 		metadata_pages += (sizeof(*mh) + mh->metalen) / getpagesize();
 
-		/* TAKEN tags end a subpage alloc. */
-		for (i = mh->metalen * CHAR_BIT/BITS_PER_PAGE - 1;
-		     i >= 0;
-		     i -= len) {
+		for (i = 0;
+		     i < mh->metalen * CHAR_BIT / BITS_PER_PAGE;
+		     i += len) {
 			switch (get_page_state(meta, i)) {
 			case FREE:
 				len = 1;
 				free++;
 				break;
-			case TAKEN:
+			case BITMAP:
 				/* Skip over this allocated part. */
 				len = BITMAP_METALEN * CHAR_BIT;
 				subpageblocks++;
@@ -716,7 +717,7 @@ void alloc_visualize(FILE *out, void *pool, unsigned long poolsize)
 	for (i = 0; i < poolsize / getpagesize(); i++) {
 		uint8_t *meta;
 		unsigned int j;
-		if (get_page_state(pool, i) != SUBPAGE)
+		if (get_page_state(pool, i) != SPECIAL)
 			continue;
 
 		memset(count, 0, sizeof(count));