tdb: use same expansion factor logic when expanding for new recovery area.

If we're expanding because the current recovery area is too small, we
expand only the amount we need.  This can quickly lead to exponential
growth when we have a slowly-expanding record (hence a
slowly-expanding transaction size).

diff --git a/ccan/tdb/io.c b/ccan/tdb/io.c
index 2262eabf93b19daf524b3876d8b7f55c1fe882ea..77f26adfe7df3eace208dee00f3f57aeaa1d28f0 100644 (file)
--- a/ccan/tdb/io.c
+++ b/ccan/tdb/io.c
@@ -303,13 +303,39 @@ static int tdb_expand_file(struct tdb_context *tdb, tdb_off_t size, tdb_off_t ad
        return 0;
 }
 
        return 0;
 }
 

+/* You need 'size', this tells you how much you should expand by. */
+tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size)
+{
+       tdb_off_t new_size, top_size;
+
+       /* limit size in order to avoid using up huge amounts of memory for
+        * in memory tdbs if an oddball huge record creeps in */
+       if (size > 100 * 1024) {
+               top_size = map_size + size * 2;
+       } else {
+               top_size = map_size + size * 100;
+       }
+
+       /* always make room for at least top_size more records, and at
+          least 25% more space. if the DB is smaller than 100MiB,
+          otherwise grow it by 10% only. */
+       if (map_size > 100 * 1024 * 1024) {
+               new_size = map_size * 1.10;
+       } else {
+               new_size = map_size * 1.25;
+       }
+
+       /* Round the database up to a multiple of the page size */
+       new_size = MAX(top_size, new_size);
+       return TDB_ALIGN(new_size, page_size) - map_size;
+}

 
 /* expand the database at least size bytes by expanding the underlying
    file and doing the mmap again if necessary */
 int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
 {
        struct tdb_record rec;
 
 /* expand the database at least size bytes by expanding the underlying
    file and doing the mmap again if necessary */
 int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
 {
        struct tdb_record rec;

-       tdb_off_t offset, new_size;     
+       tdb_off_t offset;

 
        if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
                TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
 
        if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
                TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));


@@ -319,11 +345,7 @@ int tdb_expand(struct tdb_context *tdb, tdb_off_t size)
        /* must know about any previous expansions by another process */
        tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1);
 
        /* must know about any previous expansions by another process */
        tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1);
 

-       /* always make room for at least 100 more records, and at
-           least 25% more space. Round the database up to a multiple
-           of the page size */
-       new_size = MAX(tdb->map_size + size*100, tdb->map_size * 1.25);
-       size = TDB_ALIGN(new_size, tdb->page_size) - tdb->map_size;
+       size = tdb_expand_adjust(tdb->map_size, size, tdb->page_size);

 
        if (!(tdb->flags & TDB_INTERNAL))
                tdb_munmap(tdb);
 
        if (!(tdb->flags & TDB_INTERNAL))
                tdb_munmap(tdb);

diff --git a/ccan/tdb/tdb_private.h b/ccan/tdb/tdb_private.h
index 4faf898986fb2cd324aeb90b4b25472a911ab8ba..f1c841c257975857048fc14a3fdcc46a7f919741 100644 (file)
--- a/ccan/tdb/tdb_private.h
+++ b/ccan/tdb/tdb_private.h
@@ -298,6 +298,7 @@ tdb_off_t tdb_find_lock_hash(struct tdb_context *tdb, TDB_DATA key, uint32_t has
                           struct tdb_record *rec);
 void tdb_io_init(struct tdb_context *tdb);
 int tdb_expand(struct tdb_context *tdb, tdb_off_t size);
                           struct tdb_record *rec);
 void tdb_io_init(struct tdb_context *tdb);
 int tdb_expand(struct tdb_context *tdb, tdb_off_t size);

+tdb_off_t tdb_expand_adjust(tdb_off_t map_size, tdb_off_t size, int page_size);

 int tdb_rec_free_read(struct tdb_context *tdb, tdb_off_t off,
                      struct tdb_record *rec);
 void tdb_header_hash(struct tdb_context *tdb,
 int tdb_rec_free_read(struct tdb_context *tdb, tdb_off_t off,
                      struct tdb_record *rec);
 void tdb_header_hash(struct tdb_context *tdb,

diff --git a/ccan/tdb/test/run-transaction-expand.c b/ccan/tdb/test/run-transaction-expand.c
index 463cb1069c93c8b8655b2dff9529c3dcefeca1b5..a46669d6458bb56f98b2e872ae76255da8badc7e 100644 (file)
--- a/ccan/tdb/test/run-transaction-expand.c
+++ b/ccan/tdb/test/run-transaction-expand.c
@@ -41,13 +41,10 @@ static void write_record(struct tdb_context *tdb, size_t extra_len,
        key.dsize = strlen("hi");
        key.dptr = (void *)"hi";
 
        key.dsize = strlen("hi");
        key.dptr = (void *)"hi";
 

-       data->dptr = realloc(data->dptr, data->dsize + extra_len);
-       memset(data->dptr + data->dsize, 'x', extra_len);

        data->dsize += extra_len;
        tdb_transaction_start(tdb);
        tdb_store(tdb, key, *data, TDB_REPLACE);
        tdb_transaction_commit(tdb);
        data->dsize += extra_len;
        tdb_transaction_start(tdb);
        tdb_store(tdb, key, *data, TDB_REPLACE);
        tdb_transaction_commit(tdb);

-       diag("TDB size = %zu", (size_t)tdb->map_size);

 }
 
 int main(int argc, char *argv[])
 }
 
 int main(int argc, char *argv[])


@@ -55,6 +52,8 @@ int main(int argc, char *argv[])
        struct tdb_context *tdb;
        size_t i;
        TDB_DATA data;
        struct tdb_context *tdb;
        size_t i;
        TDB_DATA data;

+       struct tdb_record rec;
+       tdb_off_t off;

 
        plan_tests(2);
        tdb = tdb_open_ex("run-transaction-expand.tdb",
 
        plan_tests(2);
        tdb = tdb_open_ex("run-transaction-expand.tdb",


@@ -63,14 +62,19 @@ int main(int argc, char *argv[])
        ok1(tdb);
 
        data.dsize = 0;
        ok1(tdb);
 
        data.dsize = 0;

-       data.dptr = NULL;
+       data.dptr = calloc(1000, getpagesize());

 
        /* Simulate a slowly growing record. */
        for (i = 0; i < 1000; i++)
                write_record(tdb, getpagesize(), &data);
 
 
        /* Simulate a slowly growing record. */
        for (i = 0; i < 1000; i++)
                write_record(tdb, getpagesize(), &data);
 

-       /* We should only be about 3 times larger than largest record. */
-       ok1(tdb->map_size < 3 * i * getpagesize());
+       tdb_ofs_read(tdb, TDB_RECOVERY_HEAD, &off);
+       tdb_read(tdb, off, &rec, sizeof(rec), DOCONV());
+       diag("TDB size = %zu, recovery = %u-%u",
+            (size_t)tdb->map_size, off, off + sizeof(rec) + rec.rec_len);
+
+       /* We should only be about 5 times larger than largest record. */
+       ok1(tdb->map_size < 6 * i * getpagesize());

        tdb_close(tdb);
        free(data.dptr);
 
        tdb_close(tdb);
        free(data.dptr);
 

diff --git a/ccan/tdb/transaction.c b/ccan/tdb/transaction.c
index 11194773647e255f1a7c8743cd69d5c85bfeda59..525e0642d2b6088bb92d1a4bfd1f8b1232d4a42f 100644 (file)
--- a/ccan/tdb/transaction.c
+++ b/ccan/tdb/transaction.c
@@ -699,7 +699,11 @@ static int tdb_recovery_allocate(struct tdb_context *tdb,
        *recovery_size = tdb_recovery_size(tdb);
 
        /* round up to a multiple of page size */
        *recovery_size = tdb_recovery_size(tdb);
 
        /* round up to a multiple of page size */

-       *recovery_max_size = TDB_ALIGN(sizeof(rec) + *recovery_size, tdb->page_size) - sizeof(rec);
+       *recovery_max_size = tdb_expand_adjust(tdb->map_size,
+                                              *recovery_size,
+                                              tdb->page_size)
+               - sizeof(rec);
+

        *recovery_offset = tdb->map_size;
        recovery_head = *recovery_offset;
 
        *recovery_offset = tdb->map_size;
        recovery_head = *recovery_offset;