From: Rusty Russell Date: Wed, 27 Apr 2011 12:09:27 +0000 (+0930) Subject: tdb2: expand more slowly. X-Git-Url: http://git.ozlabs.org/?p=ccan;a=commitdiff_plain;h=48241893712414cf3d3992a7dbe3d119925559ff tdb2: expand more slowly. We took the original expansion heuristic from TDB1, and they just fixed theirs, so copy that. Before: After: time ./growtdb-bench 250000 10 > /dev/null && ls -l /tmp/growtdb.tdb && time ./tdbtorture -s 0 && ls -l torture.tdb && ./speed --transaction 2000000 growtdb-bench.c: In function ‘main’: growtdb-bench.c:74:8: warning: ignoring return value of ‘system’, declared with attribute warn_unused_result growtdb-bench.c:108:9: warning: ignoring return value of ‘system’, declared with attribute warn_unused_result real 1m0.243s user 0m13.677s sys 0m4.336s -rw------- 1 rusty rusty 683302864 2011-04-27 21:03 /tmp/growtdb.tdb testing with 3 processes, 5000 loops, seed=0 OK real 1m24.074s user 0m0.344s sys 0m0.468s -rw------- 1 rusty rusty 836040 2011-04-27 21:04 torture.tdb Adding 2000000 records: 1015 ns (110551992 bytes) Finding 2000000 records: 641 ns (110551992 bytes) Missing 2000000 records: 445 ns (110551992 bytes) Traversing 2000000 records: 439 ns (110551992 bytes) Deleting 2000000 records: 807 ns (199517112 bytes) Re-adding 2000000 records: 851 ns (199517112 bytes) Appending 2000000 records: 1301 ns (376542552 bytes) Churning 2000000 records: 2423 ns (553641304 bytes) --- diff --git a/ccan/tdb2/free.c b/ccan/tdb2/free.c index e2b9cb18..c03680cf 100644 --- a/ccan/tdb2/free.c +++ b/ccan/tdb2/free.c @@ -713,13 +713,10 @@ enum TDB_ERROR set_header(struct tdb_context *tdb, /* Expand the database. */ static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size) { - uint64_t old_size; + uint64_t old_size, rec_size, map_size; tdb_len_t wanted; enum TDB_ERROR ecode; - /* We need room for the record header too. */ - wanted = sizeof(struct tdb_used_record) + size; - /* Need to hold a hash lock to expand DB: transactions rely on it. */ if (!(tdb->flags & TDB_NOLOCK) && !tdb->file->allrecord_lock.count && !tdb_has_hash_locks(tdb)) { @@ -727,14 +724,6 @@ static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size) "tdb_expand: must hold lock during expand"); } - /* always make room for at least 100 more records, and at - least 25% more space. */ - if (size * TDB_EXTENSION_FACTOR > tdb->file->map_size / 4) - wanted = size * TDB_EXTENSION_FACTOR; - else - wanted = tdb->file->map_size / 4; - wanted = adjust_size(0, wanted); - /* Only one person can expand file at a time. */ ecode = tdb_lock_expand(tdb, F_WRLCK); if (ecode != TDB_SUCCESS) { @@ -749,6 +738,32 @@ static enum TDB_ERROR tdb_expand(struct tdb_context *tdb, tdb_len_t size) return TDB_SUCCESS; } + /* limit size in order to avoid using up huge amounts of memory for + * in memory tdbs if an oddball huge record creeps in */ + if (size > 100 * 1024) { + rec_size = size * 2; + } else { + rec_size = size * 100; + } + + /* always make room for at least rec_size more records, and at + least 25% more space. if the DB is smaller than 100MiB, + otherwise grow it by 10% only. */ + if (old_size > 100 * 1024 * 1024) { + map_size = old_size / 10; + } else { + map_size = old_size / 4; + } + + if (map_size > rec_size) { + wanted = map_size; + } else { + wanted = rec_size; + } + + /* We need room for the record header too. */ + wanted = adjust_size(0, sizeof(struct tdb_used_record) + wanted); + ecode = tdb->methods->expand_file(tdb, wanted); if (ecode != TDB_SUCCESS) { tdb_unlock_expand(tdb, F_WRLCK); diff --git a/ccan/tdb2/tools/Makefile b/ccan/tdb2/tools/Makefile index 19bb731e..e2ad8883 100644 --- a/ccan/tdb2/tools/Makefile +++ b/ccan/tdb2/tools/Makefile @@ -2,12 +2,13 @@ OBJS:=../../tdb2.o ../../hash.o ../../tally.o CFLAGS:=-I../../.. -Wall -g -O3 #-g -pg LDFLAGS:=-L../../.. -default: tdbtorture tdbtool mktdb speed +default: tdbtorture tdbtool mktdb speed growtdb-bench tdbtorture: tdbtorture.c $(OBJS) tdbtool: tdbtool.c $(OBJS) mktdb: mktdb.c $(OBJS) speed: speed.c $(OBJS) +growtdb-bench: growtdb-bench.c $(OBJS) clean: - rm -f tdbtorture tdbtool mktdb speed + rm -f tdbtorture tdbtool mktdb speed growtdb-bench diff --git a/ccan/tdb2/tools/growtdb-bench.c b/ccan/tdb2/tools/growtdb-bench.c new file mode 100644 index 00000000..d78c413d --- /dev/null +++ b/ccan/tdb2/tools/growtdb-bench.c @@ -0,0 +1,112 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static void logfn(struct tdb_context *tdb, + enum tdb_log_level level, + const char *message, + void *data) +{ + fprintf(stderr, "tdb:%s:%s\n", tdb_name(tdb), message); +} + +int main(int argc, char *argv[]) +{ + unsigned int i, j, users, groups; + TDB_DATA idxkey, idxdata; + TDB_DATA k, d, gk; + char cmd[100]; + struct tdb_context *tdb; + enum TDB_ERROR ecode; + union tdb_attribute log; + + if (argc != 3) { + printf("Usage: growtdb-bench \n"); + exit(1); + } + users = atoi(argv[1]); + groups = atoi(argv[2]); + + sprintf(cmd, "cat /proc/%i/statm", getpid()); + + log.base.attr = TDB_ATTRIBUTE_LOG; + log.base.next = NULL; + log.log.fn = logfn; + + tdb = tdb_open("/tmp/growtdb.tdb", TDB_DEFAULT, + O_RDWR|O_CREAT|O_TRUNC, 0600, &log); + + idxkey.dptr = (unsigned char *)"User index"; + idxkey.dsize = strlen("User index"); + idxdata.dsize = 51; + idxdata.dptr = calloc(idxdata.dsize, 1); + + /* Create users. */ + k.dsize = 48; + k.dptr = calloc(k.dsize, 1); + d.dsize = 64; + d.dptr = calloc(d.dsize, 1); + + tdb_transaction_start(tdb); + for (i = 0; i < users; i++) { + memcpy(k.dptr, &i, sizeof(i)); + ecode = tdb_store(tdb, k, d, TDB_INSERT); + if (ecode != TDB_SUCCESS) + errx(1, "tdb insert failed: %s", tdb_errorstr(ecode)); + + /* This simulates a growing index record. */ + ecode = tdb_append(tdb, idxkey, idxdata); + if (ecode != TDB_SUCCESS) + errx(1, "tdb append failed: %s", tdb_errorstr(ecode)); + } + if ((ecode = tdb_transaction_commit(tdb)) != 0) + errx(1, "tdb commit1 failed: %s", tdb_errorstr(ecode)); + + if ((ecode = tdb_check(tdb, NULL, NULL)) != 0) + errx(1, "tdb_check failed after initial insert!"); + + system(cmd); + + /* Now put them all in groups: add 32 bytes to each record for + * a group. */ + gk.dsize = 48; + gk.dptr = calloc(k.dsize, 1); + gk.dptr[gk.dsize-1] = 1; + + d.dsize = 32; + for (i = 0; i < groups; i++) { + tdb_transaction_start(tdb); + /* Create the "group". */ + memcpy(gk.dptr, &i, sizeof(i)); + ecode = tdb_store(tdb, gk, d, TDB_INSERT); + if (ecode != TDB_SUCCESS) + errx(1, "tdb insert failed: %s", tdb_errorstr(ecode)); + + /* Now populate it. */ + for (j = 0; j < users; j++) { + /* Append to the user. */ + memcpy(k.dptr, &j, sizeof(j)); + if ((ecode = tdb_append(tdb, k, d)) != 0) + errx(1, "tdb append failed: %s", + tdb_errorstr(ecode)); + + /* Append to the group. */ + if ((ecode = tdb_append(tdb, gk, d)) != 0) + errx(1, "tdb append failed: %s", + tdb_errorstr(ecode)); + } + if ((ecode = tdb_transaction_commit(tdb)) != 0) + errx(1, "tdb commit2 failed: %s", tdb_errorstr(ecode)); + if ((ecode = tdb_check(tdb, NULL, NULL)) != 0) + errx(1, "tdb_check failed after iteration %i!", i); + system(cmd); + } + + return 0; +}