tdb: implement tdb_summary.

Including pretty graphs!

diff --git a/ccan/tdb/_info b/ccan/tdb/_info
index c2e5266b05f2b038276ebb021734f05d41d824a3..ec9654c6369399a37ea1f74a8eca405e947b9ab4 100644 (file)
--- a/ccan/tdb/_info
+++ b/ccan/tdb/_info
@@ -73,6 +73,7 @@ int main(int argc, char *argv[])
                return 1;
 
        if (strcmp(argv[1], "depends") == 0) {
                return 1;
 
        if (strcmp(argv[1], "depends") == 0) {

+               printf("ccan/tally\n");

                return 0;
        }
 
                return 0;
        }
 

diff --git a/ccan/tdb/summary.c b/ccan/tdb/summary.c
new file mode 100644 (file)
index 0000000..c4458d8
--- /dev/null
+++ b/ccan/tdb/summary.c
@@ -0,0 +1,190 @@
+ /* 
+   Trivial Database: human-readable summary code
+   Copyright (C) Rusty Russell 2010
+   
+   This library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 3 of the License, or (at your option) any later version.
+
+   This library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with this library; if not, see <http://www.gnu.org/licenses/>.
+*/
+#include "tdb_private.h"
+#include <ccan/tally/tally.h>
+
+#define SUMMARY_FORMAT \
+       "Size of file: %zu\n" \
+       "Number of records: %zu\n" \
+       "Smallest/average/largest records: %zu/%zu/%zu\n%s" \
+       "Smallest/average/largest padding: %zu/%zu/%zu\n%s" \
+       "Number of dead records: %zu\n" \
+       "Smallest/average/largest dead records: %zu/%zu/%zu\n%s" \
+       "Number of free records: %zu\n" \
+       "Smallest/average/largest free records: %zu/%zu/%zu\n%s" \
+       "Number of hash chains: %zu\n" \
+       "Smallest/average/largest hash chains: %zu/%zu/%zu\n%s" \
+       "Total data = %zu (%.0f%%)\n"
+
+#define HISTO_WIDTH 70
+#define HISTO_HEIGHT 20
+
+/* Slow, but should be very rare. */
+static size_t dead_space(struct tdb_context *tdb, tdb_off_t off)
+{
+       size_t len;
+
+       for (len = 0; off + len < tdb->map_size; len++) {
+               char c;
+               if (tdb->methods->tdb_read(tdb, off, &c, 1, 0))
+                       return 0;
+               if (c != 0 && c != 0x42)
+                       break;
+       }
+       return len;
+}
+
+static size_t get_hash_length(struct tdb_context *tdb, unsigned int i)
+{
+       tdb_off_t rec_ptr;
+       size_t count = 0;
+
+       if (tdb_ofs_read(tdb, TDB_HASH_TOP(i), &rec_ptr) == -1)
+               return 0;
+
+       /* keep looking until we find the right record */
+       while (rec_ptr) {
+               struct tdb_record r;
+               ++count;
+               if (tdb_rec_read(tdb, rec_ptr, &r) == -1)
+                       return 0;
+               rec_ptr = r.next;
+       }
+       return count;
+}
+
+char *tdb_summary(struct tdb_context *tdb, enum tdb_summary_flags flags)
+{
+       tdb_off_t off;
+       struct tally *freet, *used, *dead, *extra, *hash;
+       char *freeg, *usedg, *deadg, *extrag, *hashg;
+       struct tdb_record rec;
+       char *ret = NULL;
+       bool locked;
+       size_t len;
+
+       /* Read-only databases use no locking at all: it's best-effort.
+        * We may have a write lock already, so skip that case too. */
+       if (tdb->read_only || tdb->allrecord_lock.count != 0) {
+               locked = false;
+       } else {
+               if (tdb_lockall_read(tdb) == -1)
+                       return NULL;
+               locked = true;
+       }
+
+       freet = tally_new(100);
+       used = tally_new(100);
+       dead = tally_new(100);
+       extra = tally_new(100);
+       hash = tally_new(100);
+       if (!freet || !used || !dead || !extra || !hash) {
+               tdb->ecode = TDB_ERR_OOM;
+               goto unlock;
+       }
+
+       for (off = TDB_DATA_START(tdb->header.hash_size);
+            off < tdb->map_size - 1;
+            off += sizeof(rec) + rec.rec_len) {
+               if (tdb->methods->tdb_read(tdb, off, &rec, sizeof(rec),
+                                          DOCONV()) == -1)
+                       goto unlock;
+               switch (rec.magic) {
+               case TDB_MAGIC:
+                       tally_add(used, rec.key_len + rec.data_len);
+                       tally_add(extra, rec.rec_len - (rec.key_len
+                                                       + rec.data_len));
+                       break;
+               case TDB_FREE_MAGIC:
+                       tally_add(freet, sizeof(rec) + rec.rec_len);
+                       break;
+               /* If we crash after ftruncate, we can get zeroes or fill. */
+               case TDB_RECOVERY_INVALID_MAGIC:
+               case 0x42424242:
+                       rec.rec_len = dead_space(tdb, off) - sizeof(rec);
+                       /* Fall through */
+               case TDB_DEAD_MAGIC:
+                       tally_add(dead, sizeof(rec) + rec.rec_len);
+                       break;
+               default:
+                       TDB_LOG((tdb, TDB_DEBUG_ERROR,
+                                "Unexpected record magic 0x%x at offset %d\n",
+                                rec.magic, off));
+                       goto unlock;
+               }
+       }
+
+       for (off = 0; off < tdb->header.hash_size; off++)
+               tally_add(hash, get_hash_length(tdb, off));
+
+       if (flags & TDB_SUMMARY_HISTOGRAMS) {
+               freeg = tally_histogram(freet, HISTO_WIDTH, HISTO_HEIGHT);
+               usedg = tally_histogram(used, HISTO_WIDTH, HISTO_HEIGHT);
+               deadg = tally_histogram(dead, HISTO_WIDTH, HISTO_HEIGHT);
+               extrag = tally_histogram(extra, HISTO_WIDTH, HISTO_HEIGHT);
+               hashg = tally_histogram(hash, HISTO_WIDTH, HISTO_HEIGHT);
+       } else {
+               freeg = usedg = deadg = extrag = hashg = NULL;
+       }
+
+       /* 20 is max length of a %zu. */
+       len = strlen(SUMMARY_FORMAT) + 22*20 + 1
+               + (freeg ? strlen(freeg) : 0)
+               + (usedg ? strlen(usedg) : 0)
+               + (deadg ? strlen(deadg) : 0)
+               + (extrag ? strlen(extrag) : 0)
+               + (hashg ? strlen(hashg) : 0);
+       ret = malloc(len);
+       if (!ret)
+               goto unlock;
+
+       sprintf(ret, SUMMARY_FORMAT,
+               tdb->map_size,
+               tally_num(used),
+               tally_min(used), tally_mean(used), tally_max(used),
+               usedg ? usedg : "",
+               tally_min(extra), tally_mean(extra), tally_max(extra),
+               extrag ? extrag : "",
+               tally_num(dead),
+               tally_min(dead), tally_mean(dead), tally_max(dead),
+               deadg ? deadg : "",
+               tally_num(freet),
+               tally_min(freet), tally_mean(freet), tally_max(freet),
+               freeg ? freeg : "",
+               tally_num(hash),
+               tally_min(hash), tally_mean(hash), tally_max(hash),
+               hashg ? hashg : "",
+               tally_total(used, NULL),
+               tally_total(used, NULL) * 100.0 / tdb->map_size);
+
+unlock:
+       free(freeg);
+       free(usedg);
+       free(deadg);
+       free(extrag);
+       free(hashg);
+       free(freet);
+       free(used);
+       free(dead);
+       free(extra);
+       free(hash);
+       if (locked) {
+               tdb_unlockall_read(tdb);
+       }
+       return ret;
+}

diff --git a/ccan/tdb/tdb.h b/ccan/tdb/tdb.h
index 020cf050b803cd02db67d7d34aaa7cd8e042da33..070f09b4e19075e05f7bdfebff634d93fddcc89d 100644 (file)
--- a/ccan/tdb/tdb.h
+++ b/ccan/tdb/tdb.h
@@ -65,6 +65,9 @@ enum TDB_ERROR {TDB_SUCCESS=0, TDB_ERR_CORRUPT, TDB_ERR_IO, TDB_ERR_LOCK,
                TDB_ERR_NOEXIST, TDB_ERR_EINVAL, TDB_ERR_RDONLY,
                TDB_ERR_NESTING};
 
                TDB_ERR_NOEXIST, TDB_ERR_EINVAL, TDB_ERR_RDONLY,
                TDB_ERR_NESTING};
 

+/* flags for tdb_summary. Logical or to combine. */
+enum tdb_summary_flags { TDB_SUMMARY_HISTOGRAMS = 1 };
+

 /* debugging uses one of the following levels */
 enum tdb_debug_level {TDB_DEBUG_FATAL = 0, TDB_DEBUG_ERROR, 
                      TDB_DEBUG_WARNING, TDB_DEBUG_TRACE};
 /* debugging uses one of the following levels */
 enum tdb_debug_level {TDB_DEBUG_FATAL = 0, TDB_DEBUG_ERROR, 
                      TDB_DEBUG_WARNING, TDB_DEBUG_TRACE};


@@ -175,6 +178,7 @@ void tdb_dump_all(struct tdb_context *tdb);
 int tdb_printfreelist(struct tdb_context *tdb);
 int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries);
 int tdb_freelist_size(struct tdb_context *tdb);
 int tdb_printfreelist(struct tdb_context *tdb);
 int tdb_validate_freelist(struct tdb_context *tdb, int *pnum_entries);
 int tdb_freelist_size(struct tdb_context *tdb);

+char *tdb_summary(struct tdb_context *tdb, enum tdb_summary_flags flags);

 
 extern TDB_DATA tdb_null;
 
 
 extern TDB_DATA tdb_null;
 

diff --git a/ccan/tdb/tools/Makefile b/ccan/tdb/tools/Makefile
index 4dc5ceb0527f75febb027aebb70e26f45738db77..d7611a2b0fd08a709eb86a56af2b52e7cb7646ca 100644 (file)
--- a/ccan/tdb/tools/Makefile
+++ b/ccan/tdb/tools/Makefile
@@ -1,4 +1,4 @@
-LDLIBS:=../../tdb.o ../../hash.o
+LDLIBS:=../../tdb.o ../../tally.o

 CFLAGS:=-I../../.. -Wall -O3 #-g -pg
 LDFLAGS:=-L../../..
 
 CFLAGS:=-I../../.. -Wall -O3 #-g -pg
 LDFLAGS:=-L../../..
 

diff --git a/ccan/tdb/tools/tdbtool.c b/ccan/tdb/tools/tdbtool.c
index 494d1f3800fb8be5b599a1c59277fc67510be033..85d0dc5b6f91eb5928d1dd5b4eb3a37212f27a5f 100644 (file)
--- a/ccan/tdb/tools/tdbtool.c
+++ b/ccan/tdb/tools/tdbtool.c
@@ -21,7 +21,6 @@
 */
 
 #include <ccan/tdb/tdb.h>
 */
 
 #include <ccan/tdb/tdb.h>

-#include <ccan/hash/hash.h>

 #include <stdlib.h>
 #include <stdio.h>
 #include <ctype.h>
 #include <stdlib.h>
 #include <stdio.h>
 #include <ctype.h>


@@ -46,7 +45,6 @@ static int disable_mmap;
 enum commands {
        CMD_CREATE_TDB,
        CMD_OPEN_TDB,
 enum commands {
        CMD_CREATE_TDB,
        CMD_OPEN_TDB,

-       CMD_OPENJH_TDB,

        CMD_TRANSACTION_START,
        CMD_TRANSACTION_COMMIT,
        CMD_TRANSACTION_CANCEL,
        CMD_TRANSACTION_START,
        CMD_TRANSACTION_COMMIT,
        CMD_TRANSACTION_CANCEL,


@@ -80,7 +78,6 @@ typedef struct {
 COMMAND_TABLE cmd_table[] = {
        {"create",      CMD_CREATE_TDB},
        {"open",        CMD_OPEN_TDB},
 COMMAND_TABLE cmd_table[] = {
        {"create",      CMD_CREATE_TDB},
        {"open",        CMD_OPEN_TDB},

-       {"openjh",      CMD_OPENJH_TDB},

        {"transaction_start",   CMD_TRANSACTION_START},
        {"transaction_commit",  CMD_TRANSACTION_COMMIT},
        {"transaction_cancel",  CMD_TRANSACTION_CANCEL},
        {"transaction_start",   CMD_TRANSACTION_START},
        {"transaction_commit",  CMD_TRANSACTION_COMMIT},
        {"transaction_cancel",  CMD_TRANSACTION_CANCEL},


@@ -236,19 +233,14 @@ static void create_tdb(const char *tdbname)
        }
 }
 
        }
 }
 

-static unsigned int jenkins_hash(TDB_DATA *key)
-{
-       return hash_any(key->dptr, key->dsize, 0);
-}
-
-static void open_tdb(const char *tdbname, tdb_hash_func hashfn)
+static void open_tdb(const char *tdbname)

 {
        struct tdb_logging_context log_ctx;
        log_ctx.log_fn = tdb_log;
 
        if (tdb) tdb_close(tdb);
        tdb = tdb_open_ex(tdbname, 0, disable_mmap?TDB_NOMMAP:0, O_RDWR, 0600,
 {
        struct tdb_logging_context log_ctx;
        log_ctx.log_fn = tdb_log;
 
        if (tdb) tdb_close(tdb);
        tdb = tdb_open_ex(tdbname, 0, disable_mmap?TDB_NOMMAP:0, O_RDWR, 0600,

-                         &log_ctx, hashfn);
+                         &log_ctx, NULL);

        if (!tdb) {
                printf("Could not open %s: %s\n", tdbname, strerror(errno));
        }
        if (!tdb) {
                printf("Could not open %s: %s\n", tdbname, strerror(errno));
        }


@@ -420,12 +412,14 @@ static int traverse_fn(TDB_CONTEXT *the_tdb, TDB_DATA key, TDB_DATA dbuf, void *
 
 static void info_tdb(void)
 {
 
 static void info_tdb(void)
 {

-       int count;
-       total_bytes = 0;
-       if ((count = tdb_traverse(tdb, traverse_fn, NULL)) == -1)
+       char *summary = tdb_summary(tdb, TDB_SUMMARY_HISTOGRAMS);
+
+       if (!summary) {

                printf("Error = %s\n", tdb_errorstr(tdb));
                printf("Error = %s\n", tdb_errorstr(tdb));

-       else
-               printf("%d records totalling %d bytes\n", count, total_bytes);
+       } else {
+               printf("%s", summary);
+               free(summary);
+       }

 }
 
 static void speed_tdb(const char *tlimit)
 }
 
 static void speed_tdb(const char *tlimit)


@@ -586,11 +580,7 @@ static int do_command(void)
                return 0;
        case CMD_OPEN_TDB:
                bIterate = 0;
                return 0;
        case CMD_OPEN_TDB:
                bIterate = 0;

-               open_tdb(arg1, NULL);
-               return 0;
-       case CMD_OPENJH_TDB:
-               bIterate = 0;
-               open_tdb(arg1, jenkins_hash);
+               open_tdb(arg1);

                return 0;
        case CMD_SYSTEM:
                /* Shell command */
                return 0;
        case CMD_SYSTEM:
                /* Shell command */


@@ -686,7 +676,6 @@ static int do_command(void)
                        return 0;
                case CMD_CREATE_TDB:
                case CMD_OPEN_TDB:
                        return 0;
                case CMD_CREATE_TDB:
                case CMD_OPEN_TDB:

-               case CMD_OPENJH_TDB:

                case CMD_SYSTEM:
                case CMD_QUIT:
                        /*
                case CMD_SYSTEM:
                case CMD_QUIT:
                        /*