tdb: TDB_INCOMPATIBLE_HASH, to allow safe changing of default hash.

This flag to tdb_open/tdb_open_ex effects creation of a new database:
1) Uses the Jenkins lookup3 hash instead of the old gdbm hash if none is
   specified,
2) Places a non-zero field in header->rwlocks, so older versions of TDB will
   refuse to open it.

This means that the caller (ie Samba) can set this flag to safely
change the hash function.  Versions of TDB from this one on will either
use the correct hash or refuse to open (if a different hash is specified).
Older TDB versions will see the nonzero rwlocks field and refuse to open
it under any conditions.

diff --git a/ccan/tdb/check.c b/ccan/tdb/check.c
index 955f255484a720a08a07a9e33a1fde6b0036b49c..fa003b79aebabd28aed807c4f9ca1d65d3014d09 100644 (file)
--- a/ccan/tdb/check.c
+++ b/ccan/tdb/check.c
@@ -40,7 +40,7 @@ static bool tdb_check_header(struct tdb_context *tdb, tdb_off_t *recovery)
        if (hdr.version != TDB_VERSION)
                goto corrupt;
 
        if (hdr.version != TDB_VERSION)
                goto corrupt;
 

-       if (hdr.rwlocks != 0)
+       if (hdr.rwlocks != 0 && hdr.rwlocks != TDB_HASH_RWLOCK_MAGIC)

                goto corrupt;
 
        tdb_header_hash(tdb, &h1, &h2);
                goto corrupt;
 
        tdb_header_hash(tdb, &h1, &h2);

diff --git a/ccan/tdb/open.c b/ccan/tdb/open.c
index b92567b37819855a754dd4b03b94cf14fc21f800..399ed1656e4d2c88d096de9b8de2f7e7642b1cc0 100644 (file)
--- a/ccan/tdb/open.c
+++ b/ccan/tdb/open.c
@@ -71,6 +71,11 @@ static int tdb_new_database(struct tdb_context *tdb, int hash_size)
 
        tdb_header_hash(tdb, &newdb->magic1_hash, &newdb->magic2_hash);
 
 
        tdb_header_hash(tdb, &newdb->magic1_hash, &newdb->magic2_hash);
 

+       /* Make sure older tdbs (which don't check the magic hash fields)
+        * will refuse to open this TDB. */
+       if (tdb->flags & TDB_INCOMPATIBLE_HASH)
+               newdb->rwlocks = TDB_HASH_RWLOCK_MAGIC;
+

        if (tdb->flags & TDB_INTERNAL) {
                tdb->map_size = size;
                tdb->map_ptr = (char *)newdb;
        if (tdb->flags & TDB_INTERNAL) {
                tdb->map_size = size;
                tdb->map_ptr = (char *)newdb;


@@ -165,7 +170,10 @@ static bool check_header_hash(struct tdb_context *tdb,
                return false;
 
        /* Otherwise, try the other inbuilt hash. */
                return false;
 
        /* Otherwise, try the other inbuilt hash. */

-       tdb->hash_fn = tdb_jenkins_hash;
+       if (tdb->hash_fn == tdb_old_hash)
+               tdb->hash_fn = tdb_jenkins_hash;
+       else
+               tdb->hash_fn = tdb_old_hash;

        return check_header_hash(tdb, false, m1, m2);
 }
 
        return check_header_hash(tdb, false, m1, m2);
 }
 


@@ -208,7 +216,12 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
                tdb->hash_fn = hash_fn;
                hash_alg = "the user defined";
        } else {
                tdb->hash_fn = hash_fn;
                hash_alg = "the user defined";
        } else {

-               tdb->hash_fn = tdb_old_hash;
+               /* This controls what we use when creating a tdb. */
+               if (tdb->flags & TDB_INCOMPATIBLE_HASH) {
+                       tdb->hash_fn = tdb_jenkins_hash;
+               } else {
+                       tdb->hash_fn = tdb_old_hash;
+               }

                hash_alg = "either default";
        }
 
                hash_alg = "either default";
        }
 


@@ -322,13 +335,15 @@ struct tdb_context *tdb_open_ex(const char *name, int hash_size, int tdb_flags,
        if (fstat(tdb->fd, &st) == -1)
                goto fail;
 
        if (fstat(tdb->fd, &st) == -1)
                goto fail;
 

-       if (tdb->header.rwlocks != 0) {
+       if (tdb->header.rwlocks != 0 &&
+           tdb->header.rwlocks != TDB_HASH_RWLOCK_MAGIC) {

                TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_open_ex: spinlocks no longer supported\n"));
                goto fail;
        }
 
        if ((tdb->header.magic1_hash == 0) && (tdb->header.magic2_hash == 0)) {
                /* older TDB without magic hash references */
                TDB_LOG((tdb, TDB_DEBUG_ERROR, "tdb_open_ex: spinlocks no longer supported\n"));
                goto fail;
        }
 
        if ((tdb->header.magic1_hash == 0) && (tdb->header.magic2_hash == 0)) {
                /* older TDB without magic hash references */

+               tdb->hash_fn = tdb_old_hash;

        } else if (!check_header_hash(tdb, !hash_fn, &magic1, &magic2)) {
                TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_open_ex: "
                         "%s was not created with %s hash function we are using\n"
        } else if (!check_header_hash(tdb, !hash_fn, &magic1, &magic2)) {
                TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_open_ex: "
                         "%s was not created with %s hash function we are using\n"

diff --git a/ccan/tdb/tdb.h b/ccan/tdb/tdb.h
index 455cde4369951ec5642c14175d56aac563e4741a..020cf050b803cd02db67d7d34aaa7cd8e042da33 100644 (file)
--- a/ccan/tdb/tdb.h
+++ b/ccan/tdb/tdb.h
@@ -57,6 +57,7 @@ extern "C" {
 #define TDB_VOLATILE   256 /* Activate the per-hashchain freelist, default 5 */
 #define TDB_ALLOW_NESTING 512 /* Allow transactions to nest */
 #define TDB_DISALLOW_NESTING 1024 /* Disallow transactions to nest */
 #define TDB_VOLATILE   256 /* Activate the per-hashchain freelist, default 5 */
 #define TDB_ALLOW_NESTING 512 /* Allow transactions to nest */
 #define TDB_DISALLOW_NESTING 1024 /* Disallow transactions to nest */

+#define TDB_INCOMPATIBLE_HASH 2048 /* Better hashing: can't be opened by older tdb versions. */

 
 /* error codes */
 enum TDB_ERROR {TDB_SUCCESS=0, TDB_ERR_CORRUPT, TDB_ERR_IO, TDB_ERR_LOCK, 
 
 /* error codes */
 enum TDB_ERROR {TDB_SUCCESS=0, TDB_ERR_CORRUPT, TDB_ERR_IO, TDB_ERR_LOCK, 

diff --git a/ccan/tdb/tdb_private.h b/ccan/tdb/tdb_private.h
index 96fdf921f339ad8e58445b41176267aba785799c..1fdf2c6a6e8e86dce987c88f9bee689d25591cca 100644 (file)
--- a/ccan/tdb/tdb_private.h
+++ b/ccan/tdb/tdb_private.h
@@ -82,6 +82,7 @@ typedef uint32_t tdb_off_t;
 #define TDB_DEAD_MAGIC (0xFEE1DEAD)
 #define TDB_RECOVERY_MAGIC (0xf53bc0e7U)
 #define TDB_RECOVERY_INVALID_MAGIC (0x0)
 #define TDB_DEAD_MAGIC (0xFEE1DEAD)
 #define TDB_RECOVERY_MAGIC (0xf53bc0e7U)
 #define TDB_RECOVERY_INVALID_MAGIC (0x0)

+#define TDB_HASH_RWLOCK_MAGIC (0xbad1a51U)

 #define TDB_ALIGNMENT 4
 #define DEFAULT_HASH_SIZE 131
 #define FREELIST_TOP (sizeof(struct tdb_header))
 #define TDB_ALIGNMENT 4
 #define DEFAULT_HASH_SIZE 131
 #define FREELIST_TOP (sizeof(struct tdb_header))

diff --git a/ccan/tdb/test/run-incompatible.c b/ccan/tdb/test/run-incompatible.c
new file mode 100644 (file)
index 0000000..21f3b21
--- /dev/null
+++ b/ccan/tdb/test/run-incompatible.c
@@ -0,0 +1,179 @@
+#define _XOPEN_SOURCE 500
+#include <ccan/tdb/tdb.h>
+#include <ccan/tdb/io.c>
+#include <ccan/tdb/tdb.c>
+#include <ccan/tdb/lock.c>
+#include <ccan/tdb/freelist.c>
+#include <ccan/tdb/traverse.c>
+#include <ccan/tdb/transaction.c>
+#include <ccan/tdb/error.c>
+#include <ccan/tdb/open.c>
+#include <ccan/tdb/check.c>
+#include <ccan/tdb/hash.c>
+#include <ccan/tap/tap.h>
+#include <stdlib.h>
+#include <err.h>
+
+static unsigned int tdb_dumb_hash(TDB_DATA *key)
+{
+       return key->dsize;
+}
+
+static void log_fn(struct tdb_context *tdb, enum tdb_debug_level level, const char *fmt, ...)
+{
+       unsigned int *count = tdb_get_logging_private(tdb);
+       if (strstr(fmt, "hash"))
+               (*count)++;
+}
+
+static unsigned int hdr_rwlocks(const char *fname)
+{
+       struct tdb_header hdr;
+
+       int fd = open(fname, O_RDONLY);
+       if (fd == -1)
+               return -1;
+
+       if (read(fd, &hdr, sizeof(hdr)) != sizeof(hdr))
+               return -1;
+
+       close(fd);
+       return hdr.rwlocks;
+}
+
+int main(int argc, char *argv[])
+{
+       struct tdb_context *tdb;
+       unsigned int log_count, flags;
+       TDB_DATA d;
+       struct tdb_logging_context log_ctx = { log_fn, &log_count };
+
+       plan_tests(38 * 2);
+
+       for (flags = 0; flags <= TDB_CONVERT; flags += TDB_CONVERT) {
+               unsigned int rwmagic = TDB_HASH_RWLOCK_MAGIC;
+
+               if (flags & TDB_CONVERT)
+                       tdb_convert(&rwmagic, sizeof(rwmagic));
+
+               /* Create an old-style hash. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, flags,
+                                 O_CREAT|O_RDWR|O_TRUNC, 0600, &log_ctx,
+                                 NULL);
+               ok1(tdb);
+               ok1(log_count == 0);
+               d.dptr = (void *)"Hello";
+               d.dsize = 5;
+               ok1(tdb_store(tdb, d, d, TDB_INSERT) == 0);
+               tdb_close(tdb);
+
+               /* Should not have marked rwlocks field. */
+               ok1(hdr_rwlocks("run-incompatible.tdb") == 0);
+
+               /* We can still open any old-style with incompat flag. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0,
+                                 TDB_INCOMPATIBLE_HASH,
+                                 O_RDWR, 0600, &log_ctx, NULL);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_fetch(tdb, d).dsize == 5);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+
+               log_count = 0;
+               tdb = tdb_open_ex("test/jenkins-le-hash.tdb", 0, 0, O_RDONLY,
+                                 0, &log_ctx, tdb_jenkins_hash);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+
+               log_count = 0;
+               tdb = tdb_open_ex("test/jenkins-be-hash.tdb", 0, 0, O_RDONLY,
+                                 0, &log_ctx, tdb_jenkins_hash);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+
+               /* OK, now create with incompatible flag, default hash. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0,
+                                 flags|TDB_INCOMPATIBLE_HASH,
+                                 O_CREAT|O_RDWR|O_TRUNC, 0600, &log_ctx,
+                                 NULL);
+               ok1(tdb);
+               ok1(log_count == 0);
+               d.dptr = (void *)"Hello";
+               d.dsize = 5;
+               ok1(tdb_store(tdb, d, d, TDB_INSERT) == 0);
+               tdb_close(tdb);
+
+               /* Should have marked rwlocks field. */
+               ok1(hdr_rwlocks("run-incompatible.tdb") == rwmagic);
+
+               /* Cannot open with old hash. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, 0,
+                                 O_RDWR, 0600, &log_ctx, tdb_old_hash);
+               ok1(!tdb);
+               ok1(log_count == 1);
+
+               /* Can open with jenkins hash. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, 0,
+                                 O_RDWR, 0600, &log_ctx, tdb_jenkins_hash);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_fetch(tdb, d).dsize == 5);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+
+               /* Can open by letting it figure it out itself. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, 0,
+                                 O_RDWR, 0600, &log_ctx, NULL);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_fetch(tdb, d).dsize == 5);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+
+               /* We can also use incompatible hash with other hashes. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0,
+                                 flags|TDB_INCOMPATIBLE_HASH,
+                                 O_CREAT|O_RDWR|O_TRUNC, 0600, &log_ctx,
+                                 tdb_dumb_hash);
+               ok1(tdb);
+               ok1(log_count == 0);
+               d.dptr = (void *)"Hello";
+               d.dsize = 5;
+               ok1(tdb_store(tdb, d, d, TDB_INSERT) == 0);
+               tdb_close(tdb);
+
+               /* Should have marked rwlocks field. */
+               ok1(hdr_rwlocks("run-incompatible.tdb") == rwmagic);
+
+               /* It should not open if we don't specify. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, 0, O_RDWR, 0,
+                                 &log_ctx, NULL);
+               ok1(!tdb);
+               ok1(log_count == 1);
+
+               /* Should reopen with correct hash. */
+               log_count = 0;
+               tdb = tdb_open_ex("run-incompatible.tdb", 0, 0, O_RDWR, 0,
+                                 &log_ctx, tdb_dumb_hash);
+               ok1(tdb);
+               ok1(log_count == 0);
+               ok1(tdb_fetch(tdb, d).dsize == 5);
+               ok1(tdb_check(tdb, NULL, NULL) == 0);
+               tdb_close(tdb);
+       }
+
+       return exit_status();
+}