From 67aef98e2e4409b69ba668a2a4a9f0757ddc7ca7 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Tue, 4 Aug 2009 15:44:16 +0930 Subject: [PATCH] Make TDB 32 bit clean (not just 31 bit) for systems with 64 bit file offsets. --- ccan/tdb/tdb_private.h | 1 + ccan/tdb/test/run-3G-file.c | 127 ++++++++++++++++++++++++++++++++++++ ccan/tdb/traverse.c | 30 ++++++--- 3 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 ccan/tdb/test/run-3G-file.c diff --git a/ccan/tdb/tdb_private.h b/ccan/tdb/tdb_private.h index 727f5c2a..42770c35 100644 --- a/ccan/tdb/tdb_private.h +++ b/ccan/tdb/tdb_private.h @@ -34,6 +34,7 @@ #include "system/wait.h" #else #define _XOPEN_SOURCE 500 +#define _FILE_OFFSET_BITS 64 #include #include #include diff --git a/ccan/tdb/test/run-3G-file.c b/ccan/tdb/test/run-3G-file.c new file mode 100644 index 00000000..d19decc7 --- /dev/null +++ b/ccan/tdb/test/run-3G-file.c @@ -0,0 +1,127 @@ +/* We need this otherwise fcntl locking fails. */ +#define _FILE_OFFSET_BITS 64 +#define _XOPEN_SOURCE 500 +#include "tdb/tdb.h" +#include "tdb/io.c" +#include "tdb/tdb.c" +#include "tdb/lock.c" +#include "tdb/freelist.c" +#include "tdb/traverse.c" +#include "tdb/transaction.c" +#include "tdb/error.c" +#include "tdb/open.c" +#include "tap/tap.h" +#include +#include + +static int tdb_expand_file_sparse(struct tdb_context *tdb, + tdb_off_t size, + tdb_off_t addition) +{ + if (tdb->read_only || tdb->traverse_read) { + tdb->ecode = TDB_ERR_RDONLY; + return -1; + } + + if (ftruncate(tdb->fd, size+addition) == -1) { + char b = 0; + ssize_t written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + if (written == 0) { + /* try once more, potentially revealing errno */ + written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + } + if (written == 0) { + /* again - give up, guessing errno */ + errno = ENOSPC; + } + if (written != 1) { + TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %d failed (%s)\n", + size+addition, strerror(errno))); + return -1; + } + } + + return 0; +} + +static const struct tdb_methods large_io_methods = { + tdb_read, + tdb_write, + tdb_next_hash_chain, + tdb_oob, + tdb_expand_file_sparse, + tdb_brlock +}; + +static int test_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data, + void *_data) +{ + TDB_DATA *expect = _data; + ok1(key.dsize == strlen("hi")); + ok1(memcmp(key.dptr, "hi", strlen("hi")) == 0); + ok1(data.dsize == expect->dsize); + ok1(memcmp(data.dptr, expect->dptr, data.dsize) == 0); + return 0; +} + +int main(int argc, char *argv[]) +{ + struct tdb_context *tdb; + TDB_DATA key, orig_data, data; + uint32_t hash; + tdb_off_t rec_ptr; + struct list_struct rec; + + plan_tests(24); + tdb = tdb_open("/tmp/test.tdb", 1024, TDB_CLEAR_IF_FIRST, + O_CREAT|O_TRUNC|O_RDWR, 0600); + + ok1(tdb); + tdb->methods = &large_io_methods; + + /* Enlarge the file (internally multiplies by 100). */ + ok1(tdb_expand(tdb, 30000000) == 0); + + /* Put an entry in, and check it. */ + key.dsize = strlen("hi"); + key.dptr = (void *)"hi"; + orig_data.dsize = strlen("world"); + orig_data.dptr = (void *)"world"; + + ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0); + + data = tdb_fetch(tdb, key); + ok1(data.dsize == strlen("world")); + ok1(memcmp(data.dptr, "world", strlen("world")) == 0); + free(data.dptr); + + /* That currently fills at the end, make sure that's true. */ + hash = tdb->hash_fn(&key); + rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_RDLCK, &rec); + ok1(rec_ptr); + ok1(rec_ptr > 2U*1024*1024*1024); + tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK); + + /* Traverse must work. */ + ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1); + + /* Delete should work. */ + ok1(tdb_delete(tdb, key) == 0); + + ok1(tdb_traverse(tdb, test_traverse, NULL) == 0); + + /* Transactions should work. */ + ok1(tdb_transaction_start(tdb) == 0); + ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0); + + data = tdb_fetch(tdb, key); + ok1(data.dsize == strlen("world")); + ok1(memcmp(data.dptr, "world", strlen("world")) == 0); + free(data.dptr); + ok1(tdb_transaction_commit(tdb) == 0); + + ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1); + tdb_close(tdb); + + return exit_status(); +} diff --git a/ccan/tdb/traverse.c b/ccan/tdb/traverse.c index 4b3a3162..8d808206 100644 --- a/ccan/tdb/traverse.c +++ b/ccan/tdb/traverse.c @@ -27,8 +27,11 @@ #include "tdb_private.h" -/* Uses traverse lock: 0 = finish, -1 = error, other = record offset */ -static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock, +#define TDB_NEXT_LOCK_ERR ((tdb_off_t)-1) + +/* Uses traverse lock: 0 = finish, TDB_NEXT_LOCK_ERR = error, + other = record offset */ +static tdb_off_t tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock, struct list_struct *rec) { int want_next = (tlock->off != 0); @@ -71,7 +74,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc } if (tdb_lock(tdb, tlock->hash, tlock->lock_rw) == -1) - return -1; + return TDB_NEXT_LOCK_ERR; /* No previous record? Start at top of chain. */ if (!tlock->off) { @@ -99,6 +102,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc /* Detect infinite loops. From "Shlomi Yaakobovich" . */ if (tlock->off == rec->next) { + tdb->ecode = TDB_ERR_CORRUPT; TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: loop detected.\n")); goto fail; } @@ -128,7 +132,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc tlock->off = 0; if (tdb_unlock(tdb, tlock->hash, tlock->lock_rw) != 0) TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: On error unlock failed!\n")); - return -1; + return TDB_NEXT_LOCK_ERR; } /* traverse the entire database - calling fn(tdb, key, data) on each element. @@ -142,7 +146,8 @@ static int tdb_traverse_internal(struct tdb_context *tdb, { TDB_DATA key, dbuf; struct list_struct rec; - int ret, count = 0; + int ret = 0, count = 0; + tdb_off_t off; /* This was in the initializaton, above, but the IRIX compiler * did not like it. crh @@ -153,7 +158,11 @@ static int tdb_traverse_internal(struct tdb_context *tdb, tdb->travlocks.next = tl; /* tdb_next_lock places locks on the record returned, and its chain */ - while ((ret = tdb_next_lock(tdb, tl, &rec)) > 0) { + while ((off = tdb_next_lock(tdb, tl, &rec)) != 0) { + if (off == TDB_NEXT_LOCK_ERR) { + ret = -1; + goto out; + } count++; /* now read the full record */ key.dptr = tdb_alloc_read(tdb, tl->off + sizeof(rec), @@ -181,7 +190,6 @@ static int tdb_traverse_internal(struct tdb_context *tdb, if (fn && fn(tdb, key, dbuf, private_data)) { /* They want us to terminate traversal */ tdb_trace_ret(tdb, "tdb_traverse_end", count); - ret = count; if (tdb_unlock_record(tdb, tl->off) != 0) { TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: unlock_record failed!\n"));; ret = -1; @@ -263,6 +271,7 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb) { TDB_DATA key; struct list_struct rec; + tdb_off_t off; /* release any old lock */ if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0) @@ -271,7 +280,8 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb) tdb->travlocks.lock_rw = F_RDLCK; /* Grab first record: locks chain and returned record. */ - if (tdb_next_lock(tdb, &tdb->travlocks, &rec) <= 0) { + off = tdb_next_lock(tdb, &tdb->travlocks, &rec); + if (off == 0 || off == TDB_NEXT_LOCK_ERR) { tdb_trace_retrec(tdb, "tdb_firstkey", tdb_null); return tdb_null; } @@ -294,6 +304,7 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey) TDB_DATA key = tdb_null; struct list_struct rec; unsigned char *k = NULL; + tdb_off_t off; /* Is locked key the old key? If so, traverse will be reliable. */ if (tdb->travlocks.off) { @@ -337,7 +348,8 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey) /* Grab next record: locks chain and returned record, unlocks old record */ - if (tdb_next_lock(tdb, &tdb->travlocks, &rec) > 0) { + off = tdb_next_lock(tdb, &tdb->travlocks, &rec); + if (off != TDB_NEXT_LOCK_ERR && off != 0) { key.dsize = rec.key_len; key.dptr = tdb_alloc_read(tdb, tdb->travlocks.off+sizeof(rec), key.dsize); -- 2.39.2