From 1c7178a0a9173ff99a581475d8a9089deadcccb2 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Fri, 24 Sep 2010 17:27:51 +0930 Subject: [PATCH] tdb: implement tdb_summary. Including pretty graphs! --- ccan/tdb/_info | 1 + ccan/tdb/summary.c | 190 +++++++++++++++++++++++++++++++++++++++ ccan/tdb/tdb.h | 4 + ccan/tdb/tools/Makefile | 2 +- ccan/tdb/tools/tdbtool.c | 31 +++---- 5 files changed, 206 insertions(+), 22 deletions(-) create mode 100644 ccan/tdb/summary.c diff --git a/ccan/tdb/_info b/ccan/tdb/_info index c2e5266b..ec9654c6 100644 --- a/ccan/tdb/_info +++ b/ccan/tdb/_info @@ -73,6 +73,7 @@ int main(int argc, char *argv[]) return 1; if (strcmp(argv[1], "depends") == 0) { + printf("ccan/tally\n"); return 0; } diff --git a/ccan/tdb/summary.c b/ccan/tdb/summary.c new file mode 100644 index 00000000..c4458d85 --- /dev/null +++ b/ccan/tdb/summary.c @@ -0,0 +1,190 @@ + /* + Trivial Database: human-readable summary code + Copyright (C) Rusty Russell 2010 + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 3 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, see . +*/ +#include "tdb_private.h" +#include + +#define SUMMARY_FORMAT \ + "Size of file: %zu\n" \ + "Number of records: %zu\n" \ + "Smallest/average/largest records: %zu/%zu/%zu\n%s" \ + "Smallest/average/largest padding: %zu/%zu/%zu\n%s" \ + "Number of dead records: %zu\n" \ + "Smallest/average/largest dead records: %zu/%zu/%zu\n%s" \ + "Number of free records: %zu\n" \ + "Smallest/average/largest free records: %zu/%zu/%zu\n%s" \ + "Number of hash chains: %zu\n" \ + "Smallest/average/largest hash chains: %zu/%zu/%zu\n%s" \ + "Total data = %zu (%.0f%%)\n" + +#define HISTO_WIDTH 70 +#define HISTO_HEIGHT 20 + +/* Slow, but should be very rare. */ +static size_t dead_space(struct tdb_context *tdb, tdb_off_t off) +{ + size_t len; + + for (len = 0; off + len < tdb->map_size; len++) { + char c; + if (tdb->methods->tdb_read(tdb, off, &c, 1, 0)) + return 0; + if (c != 0 && c != 0x42) + break; + } + return len; +} + +static size_t get_hash_length(struct tdb_context *tdb, unsigned int i) +{ + tdb_off_t rec_ptr; + size_t count = 0; + + if (tdb_ofs_read(tdb, TDB_HASH_TOP(i), &rec_ptr) == -1) + return 0; + + /* keep looking until we find the right record */ + while (rec_ptr) { + struct tdb_record r; + ++count; + if (tdb_rec_read(tdb, rec_ptr, &r) == -1) + return 0; + rec_ptr = r.next; + } + return count; +} + +char *tdb_summary(struct tdb_context *tdb, enum tdb_summary_flags flags) +{ + tdb_off_t off; + struct tally *freet, *used, *dead, *extra, *hash; + char *freeg, *usedg, *deadg, *extrag, *hashg; + struct tdb_record rec; + char *ret = NULL; + bool locked; + size_t len; + + /* Read-only databases use no locking at all: it's best-effort. + * We may have a write lock already, so skip that case too. */ + if (tdb->read_only || tdb->allrecord_lock.count != 0) { + locked = false; + } else { + if (tdb_lockall_read(tdb) == -1) + return NULL; + locked = true; + } + + freet = tally_new(100); + used = tally_new(100); + dead = tally_new(100); + extra = tally_new(100); + hash = tally_new(100); + if (!freet || !used || !dead || !extra || !hash) { + tdb->ecode = TDB_ERR_OOM; + goto unlock; + } + + for (off = TDB_DATA_START(tdb->header.hash_size); + off < tdb->map_size - 1; + off += sizeof(rec) + rec.rec_len) { + if (tdb->methods->tdb_read(tdb, off, &rec, sizeof(rec), + DOCONV()) == -1) + goto unlock; + switch (rec.magic) { + case TDB_MAGIC: + tally_add(used, rec.key_len + rec.data_len); + tally_add(extra, rec.rec_len - (rec.key_len + + rec.data_len)); + break; + case TDB_FREE_MAGIC: + tally_add(freet, sizeof(rec) + rec.rec_len); + break; + /* If we crash after ftruncate, we can get zeroes or fill. */ + case TDB_RECOVERY_INVALID_MAGIC: + case 0x42424242: + rec.rec_len = dead_space(tdb, off) - sizeof(rec); + /* Fall through */ + case TDB_DEAD_MAGIC: + tally_add(dead, sizeof(rec) + rec.rec_len); + break; + default: + TDB_LOG((tdb, TDB_DEBUG_ERROR, + "Unexpected record magic 0x%x at offset %d\n", + rec.magic, off)); + goto unlock; + } + } + + for (off = 0; off < tdb->header.hash_size; off++) + tally_add(hash, get_hash_length(tdb, off)); + + if (flags & TDB_SUMMARY_HISTOGRAMS) { + freeg = tally_histogram(freet, HISTO_WIDTH, HISTO_HEIGHT); + usedg = tally_histogram(used, HISTO_WIDTH, HISTO_HEIGHT); + deadg = tally_histogram(dead, HISTO_WIDTH, HISTO_HEIGHT); + extrag = tally_histogram(extra, HISTO_WIDTH, HISTO_HEIGHT); + hashg = tally_histogram(hash, HISTO_WIDTH, HISTO_HEIGHT); + } else { + freeg = usedg = deadg = extrag = hashg = NULL; + } + + /* 20 is max length of a %zu. */ + len = strlen(SUMMARY_FORMAT) + 22*20 + 1 + + (freeg ? strlen(freeg) : 0) + + (usedg ? strlen(usedg) : 0) + + (deadg ? strlen(deadg) : 0) + + (extrag ? strlen(extrag) : 0) + + (hashg ? strlen(hashg) : 0); + ret = malloc(len); + if (!ret) + goto unlock; + + sprintf(ret, SUMMARY_FORMAT, + tdb->map_size, + tally_num(used), + tally_min(used), tally_mean(used), tally_max(used), + usedg ? usedg : "", + tally_min(extra), tally_mean(extra), tally_max(extra), + extrag ? extrag : "", + tally_num(dead), + tally_min(dead), tally_mean(dead), tally_max(dead), + deadg ? deadg : "", + tally_num(freet), + tally_min(freet), tally_mean(freet), tally_max(freet), + freeg ? freeg : "", + tally_num(hash), + tally_min(hash), tally_mean(hash), tally_max(hash), + hashg ? hashg : "", + tally_total(used, NULL), + tally_total(used, NULL) * 100.0 / tdb->map_size); + +unlock: + free(freeg); + free(usedg); + free(deadg); + free(extrag); + free(hashg); + free(freet); + free(used); + free(dead); + free(extra); + free(hash); + if (locked) { + tdb_unlockall_read(tdb); + } + return ret; +} diff --git a/ccan/tdb/tdb.h b/ccan/tdb/tdb.h index 020cf050..070f09b4 100644 --- a/ccan/tdb/tdb.h +++ b/ccan/tdb/tdb.h @@ -65,6 +65,9 @@ enum TDB_ERROR {TDB_SUCCESS=0, TDB_ERR_CORRUPT, TDB_ERR_IO, TDB_ERR_LOCK, TDB_ERR_NOEXIST, TDB_ERR_EINVAL, TDB_ERR_RDONLY, TDB_ERR_NESTING}; +/* flags for tdb_summary. Logical or to combine. */ +enum tdb_summary_flags { TDB_SUMMARY_HISTOGRAMS = 1 }; + /* debugging uses one of the following levels */ enum tdb_debug_level {TDB_DEBUG_FATAL = 0, TDB_DEBUG_ERROR, TDB_DEBUG_WARNING, TDB_DEBUG_TRACE}; @@ -175,6 +178,7 @@ void tdb_dump_all(struct tdb_context *tdb); int tdb_printfreelist(struct tdb_context *tdb); int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries); int tdb_freelist_size(struct tdb_context *tdb); +char *tdb_summary(struct tdb_context *tdb, enum tdb_summary_flags flags); extern TDB_DATA tdb_null; diff --git a/ccan/tdb/tools/Makefile b/ccan/tdb/tools/Makefile index 4dc5ceb0..d7611a2b 100644 --- a/ccan/tdb/tools/Makefile +++ b/ccan/tdb/tools/Makefile @@ -1,4 +1,4 @@ -LDLIBS:=../../tdb.o ../../hash.o +LDLIBS:=../../tdb.o ../../tally.o CFLAGS:=-I../../.. -Wall -O3 #-g -pg LDFLAGS:=-L../../.. diff --git a/ccan/tdb/tools/tdbtool.c b/ccan/tdb/tools/tdbtool.c index 494d1f38..85d0dc5b 100644 --- a/ccan/tdb/tools/tdbtool.c +++ b/ccan/tdb/tools/tdbtool.c @@ -21,7 +21,6 @@ */ #include -#include #include #include #include @@ -46,7 +45,6 @@ static int disable_mmap; enum commands { CMD_CREATE_TDB, CMD_OPEN_TDB, - CMD_OPENJH_TDB, CMD_TRANSACTION_START, CMD_TRANSACTION_COMMIT, CMD_TRANSACTION_CANCEL, @@ -80,7 +78,6 @@ typedef struct { COMMAND_TABLE cmd_table[] = { {"create", CMD_CREATE_TDB}, {"open", CMD_OPEN_TDB}, - {"openjh", CMD_OPENJH_TDB}, {"transaction_start", CMD_TRANSACTION_START}, {"transaction_commit", CMD_TRANSACTION_COMMIT}, {"transaction_cancel", CMD_TRANSACTION_CANCEL}, @@ -236,19 +233,14 @@ static void create_tdb(const char *tdbname) } } -static unsigned int jenkins_hash(TDB_DATA *key) -{ - return hash_any(key->dptr, key->dsize, 0); -} - -static void open_tdb(const char *tdbname, tdb_hash_func hashfn) +static void open_tdb(const char *tdbname) { struct tdb_logging_context log_ctx; log_ctx.log_fn = tdb_log; if (tdb) tdb_close(tdb); tdb = tdb_open_ex(tdbname, 0, disable_mmap?TDB_NOMMAP:0, O_RDWR, 0600, - &log_ctx, hashfn); + &log_ctx, NULL); if (!tdb) { printf("Could not open %s: %s\n", tdbname, strerror(errno)); } @@ -420,12 +412,14 @@ static int traverse_fn(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void * static void info_tdb(void) { - int count; - total_bytes = 0; - if ((count = tdb_traverse(tdb, traverse_fn, NULL)) == -1) + char *summary = tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS); + + if (!summary) { printf("Error = %s\n", tdb_errorstr(tdb)); - else - printf("%d records totalling %d bytes\n", count, total_bytes); + } else { + printf("%s", summary); + free(summary); + } } static void speed_tdb(const char *tlimit) @@ -586,11 +580,7 @@ static int do_command(void) return 0; case CMD_OPEN_TDB: bIterate = 0; - open_tdb(arg1, NULL); - return 0; - case CMD_OPENJH_TDB: - bIterate = 0; - open_tdb(arg1, jenkins_hash); + open_tdb(arg1); return 0; case CMD_SYSTEM: /* Shell command */ @@ -686,7 +676,6 @@ static int do_command(void) return 0; case CMD_CREATE_TDB: case CMD_OPEN_TDB: - case CMD_OPENJH_TDB: case CMD_SYSTEM: case CMD_QUIT: /* -- 2.39.2