tdb2: tdb1: use same expansion factor logic when expanding for new recovery area.

If we're expanding because the current recovery area is too small, we
expand only the amount we need.  This can quickly lead to exponential
growth when we have a slowly-expanding record (hence a
slowly-expanding transaction size).

(Syncs with tdb1 SAMBA patch 3a2a755e3380a8f81374009d463cd06161352507)

diff --git a/ccan/tdb2/tdb1_io.c b/ccan/tdb2/tdb1_io.c
index ceae5285a7a1bb0e4607cca4ec8d81ac45aa994a..488f3d847837e2009b4eee2968932c3dbbfaa080 100644 (file)
--- a/ccan/tdb2/tdb1_io.c
+++ b/ccan/tdb2/tdb1_io.c
@@ -330,42 +330,51 @@ static int tdb1_expand_file(struct tdb_context *tdb, tdb1_off_t size, tdb1_off_t
 }
 
 
 }
 
 

-/* expand the database at least size bytes by expanding the underlying
-   file and doing the mmap again if necessary */
-int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size)
+/* You need 'size', this tells you how much you should expand by. */
+tdb1_off_t tdb1_expand_adjust(tdb1_off_t map_size, tdb1_off_t size, int page_size)

 {
 {

-       struct tdb1_record rec;
-       tdb1_off_t offset, new_size, top_size, map_size;
-
-       if (tdb1_lock(tdb, -1, F_WRLCK) == -1) {
-               tdb_logerr(tdb, tdb->last_error, TDB_LOG_ERROR,
-                          "lock failed in tdb1_expand");
-               return -1;
-       }
-
-       /* must know about any previous expansions by another process */
-       tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size, 1, 1);
+       tdb1_off_t new_size, top_size;

 
        /* limit size in order to avoid using up huge amounts of memory for
         * in memory tdbs if an oddball huge record creeps in */
        if (size > 100 * 1024) {
 
        /* limit size in order to avoid using up huge amounts of memory for
         * in memory tdbs if an oddball huge record creeps in */
        if (size > 100 * 1024) {

-               top_size = tdb->file->map_size + size * 2;
+               top_size = map_size + size * 2;

        } else {
        } else {

-               top_size = tdb->file->map_size + size * 100;
+               top_size = map_size + size * 100;

        }
 
        /* always make room for at least top_size more records, and at
           least 25% more space. if the DB is smaller than 100MiB,
           otherwise grow it by 10% only. */
        }
 
        /* always make room for at least top_size more records, and at
           least 25% more space. if the DB is smaller than 100MiB,
           otherwise grow it by 10% only. */

-       if (tdb->file->map_size > 100 * 1024 * 1024) {
-               map_size = tdb->file->map_size * 1.10;
+       if (map_size > 100 * 1024 * 1024) {
+               new_size = map_size * 1.10;

        } else {
        } else {

-               map_size = tdb->file->map_size * 1.25;
+               new_size = map_size * 1.25;

        }
 
        /* Round the database up to a multiple of the page size */
        }
 
        /* Round the database up to a multiple of the page size */

-       new_size = MAX(top_size, map_size);
-       size = TDB1_ALIGN(new_size, tdb->tdb1.page_size) - tdb->file->map_size;
+       new_size = MAX(top_size, new_size);
+       return TDB1_ALIGN(new_size, page_size) - map_size;
+}
+
+/* expand the database at least size bytes by expanding the underlying
+   file and doing the mmap again if necessary */
+int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size)
+{
+       struct tdb1_record rec;
+       tdb1_off_t offset;
+
+       if (tdb1_lock(tdb, -1, F_WRLCK) == -1) {
+               tdb_logerr(tdb, tdb->last_error, TDB_LOG_ERROR,
+                          "lock failed in tdb1_expand");
+               return -1;
+       }
+
+       /* must know about any previous expansions by another process */
+       tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size, 1, 1);
+
+       size = tdb1_expand_adjust(tdb->file->map_size, size,
+                                 tdb->tdb1.page_size);

 
        if (!(tdb->flags & TDB_INTERNAL))
                tdb1_munmap(tdb);
 
        if (!(tdb->flags & TDB_INTERNAL))
                tdb1_munmap(tdb);

diff --git a/ccan/tdb2/tdb1_private.h b/ccan/tdb2/tdb1_private.h
index fe2681ceec582112dd9f584c3cbaf221fcc92e8c..cb22b9f332cb3671ab8274574ba4829f24a14b21 100644 (file)
--- a/ccan/tdb2/tdb1_private.h
+++ b/ccan/tdb2/tdb1_private.h
@@ -168,6 +168,7 @@ tdb1_off_t tdb1_find_lock_hash(struct tdb_context *tdb, TDB_DATA key, uint32_t h
                           struct tdb1_record *rec);
 void tdb1_io_init(struct tdb_context *tdb);
 int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size);
                           struct tdb1_record *rec);
 void tdb1_io_init(struct tdb_context *tdb);
 int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size);

+tdb1_off_t tdb1_expand_adjust(tdb1_off_t map_size, tdb1_off_t size, int page_size);

 int tdb1_rec_free_read(struct tdb_context *tdb, tdb1_off_t off,
                      struct tdb1_record *rec);
 bool tdb1_write_all(int fd, const void *buf, size_t count);
 int tdb1_rec_free_read(struct tdb_context *tdb, tdb1_off_t off,
                      struct tdb1_record *rec);
 bool tdb1_write_all(int fd, const void *buf, size_t count);

diff --git a/ccan/tdb2/tdb1_transaction.c b/ccan/tdb2/tdb1_transaction.c
index c4a2b10b20759685112524f6a101e69964d3ec9f..9cb95235662d7864e3739e75f9d86ee1d8760715 100644 (file)
--- a/ccan/tdb2/tdb1_transaction.c
+++ b/ccan/tdb2/tdb1_transaction.c
@@ -745,8 +745,11 @@ static int tdb1_recovery_allocate(struct tdb_context *tdb,
        *recovery_size = tdb1_recovery_size(tdb);
 
        /* round up to a multiple of page size */
        *recovery_size = tdb1_recovery_size(tdb);
 
        /* round up to a multiple of page size */

-       *recovery_max_size = TDB1_ALIGN(sizeof(rec) + *recovery_size,
-                                       tdb->tdb1.page_size) - sizeof(rec);
+       *recovery_max_size = tdb1_expand_adjust(tdb->file->map_size,
+                                              *recovery_size,
+                                              tdb->tdb1.page_size)
+               - sizeof(rec);
+

        *recovery_offset = tdb->file->map_size;
        recovery_head = *recovery_offset;
 
        *recovery_offset = tdb->file->map_size;
        recovery_head = *recovery_offset;