Make TDB 32 bit clean (not just 31 bit) for systems with 64 bit file offsets.
authorRusty Russell <rusty@rustcorp.com.au>
Tue, 4 Aug 2009 06:14:16 +0000 (15:44 +0930)
committerRusty Russell <rusty@rustcorp.com.au>
Tue, 4 Aug 2009 06:14:16 +0000 (15:44 +0930)
ccan/tdb/tdb_private.h
ccan/tdb/test/run-3G-file.c [new file with mode: 0644]
ccan/tdb/traverse.c

index 727f5c2a2f1641efd7613418a313ba03da20c75c..42770c353b6fa106eff6f8615ab2b2f7235dad6d 100644 (file)
@@ -34,6 +34,7 @@
 #include "system/wait.h"
 #else
 #define _XOPEN_SOURCE 500
+#define _FILE_OFFSET_BITS 64
 #include <stdint.h>
 #include <stdbool.h>
 #include <stdlib.h>
diff --git a/ccan/tdb/test/run-3G-file.c b/ccan/tdb/test/run-3G-file.c
new file mode 100644 (file)
index 0000000..d19decc
--- /dev/null
@@ -0,0 +1,127 @@
+/* We need this otherwise fcntl locking fails. */
+#define _FILE_OFFSET_BITS 64
+#define _XOPEN_SOURCE 500
+#include "tdb/tdb.h"
+#include "tdb/io.c"
+#include "tdb/tdb.c"
+#include "tdb/lock.c"
+#include "tdb/freelist.c"
+#include "tdb/traverse.c"
+#include "tdb/transaction.c"
+#include "tdb/error.c"
+#include "tdb/open.c"
+#include "tap/tap.h"
+#include <stdlib.h>
+#include <err.h>
+
+static int tdb_expand_file_sparse(struct tdb_context *tdb,
+                                 tdb_off_t size,
+                                 tdb_off_t addition)
+{
+       if (tdb->read_only || tdb->traverse_read) {
+               tdb->ecode = TDB_ERR_RDONLY;
+               return -1;
+       }
+
+       if (ftruncate(tdb->fd, size+addition) == -1) {
+               char b = 0;
+               ssize_t written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
+               if (written == 0) {
+                       /* try once more, potentially revealing errno */
+                       written = pwrite(tdb->fd,  &b, 1, (size+addition) - 1);
+               }
+               if (written == 0) {
+                       /* again - give up, guessing errno */
+                       errno = ENOSPC;
+               }
+               if (written != 1) {
+                       TDB_LOG((tdb, TDB_DEBUG_FATAL, "expand_file to %d failed (%s)\n", 
+                                size+addition, strerror(errno)));
+                       return -1;
+               }
+       }
+
+       return 0;
+}
+
+static const struct tdb_methods large_io_methods = {
+       tdb_read,
+       tdb_write,
+       tdb_next_hash_chain,
+       tdb_oob,
+       tdb_expand_file_sparse,
+       tdb_brlock
+};
+
+static int test_traverse(struct tdb_context *tdb, TDB_DATA key, TDB_DATA data,
+                        void *_data)
+{
+       TDB_DATA *expect = _data;
+       ok1(key.dsize == strlen("hi"));
+       ok1(memcmp(key.dptr, "hi", strlen("hi")) == 0);
+       ok1(data.dsize == expect->dsize);
+       ok1(memcmp(data.dptr, expect->dptr, data.dsize) == 0);
+       return 0;
+}
+
+int main(int argc, char *argv[])
+{
+       struct tdb_context *tdb;
+       TDB_DATA key, orig_data, data;
+       uint32_t hash;
+       tdb_off_t rec_ptr;
+       struct list_struct rec;
+
+       plan_tests(24);
+       tdb = tdb_open("/tmp/test.tdb", 1024, TDB_CLEAR_IF_FIRST,
+                      O_CREAT|O_TRUNC|O_RDWR, 0600);
+
+       ok1(tdb);
+       tdb->methods = &large_io_methods;
+
+       /* Enlarge the file (internally multiplies by 100). */
+       ok1(tdb_expand(tdb, 30000000) == 0);
+
+       /* Put an entry in, and check it. */
+       key.dsize = strlen("hi");
+       key.dptr = (void *)"hi";
+       orig_data.dsize = strlen("world");
+       orig_data.dptr = (void *)"world";
+
+       ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
+
+       data = tdb_fetch(tdb, key);
+       ok1(data.dsize == strlen("world"));
+       ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
+       free(data.dptr);
+
+       /* That currently fills at the end, make sure that's true. */
+       hash = tdb->hash_fn(&key);
+       rec_ptr = tdb_find_lock_hash(tdb, key, hash, F_RDLCK, &rec);
+       ok1(rec_ptr);
+       ok1(rec_ptr > 2U*1024*1024*1024);
+       tdb_unlock(tdb, BUCKET(rec.full_hash), F_RDLCK);
+
+       /* Traverse must work. */
+       ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
+
+       /* Delete should work. */
+       ok1(tdb_delete(tdb, key) == 0);
+
+       ok1(tdb_traverse(tdb, test_traverse, NULL) == 0);
+
+       /* Transactions should work. */
+       ok1(tdb_transaction_start(tdb) == 0);
+       ok1(tdb_store(tdb, key, orig_data, TDB_INSERT) == 0);
+
+       data = tdb_fetch(tdb, key);
+       ok1(data.dsize == strlen("world"));
+       ok1(memcmp(data.dptr, "world", strlen("world")) == 0);
+       free(data.dptr);
+       ok1(tdb_transaction_commit(tdb) == 0);
+
+       ok1(tdb_traverse(tdb, test_traverse, &orig_data) == 1);
+       tdb_close(tdb);
+
+       return exit_status();
+}
index 4b3a316229c2059ef1977b9466b2d5d5fe34473d..8d808206f96b6fc3dcf1ee70a7d7f4687f5282a3 100644 (file)
 
 #include "tdb_private.h"
 
-/* Uses traverse lock: 0 = finish, -1 = error, other = record offset */
-static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock,
+#define TDB_NEXT_LOCK_ERR ((tdb_off_t)-1)
+
+/* Uses traverse lock: 0 = finish, TDB_NEXT_LOCK_ERR = error,
+   other = record offset */
+static tdb_off_t tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tlock,
                         struct list_struct *rec)
 {
        int want_next = (tlock->off != 0);
@@ -71,7 +74,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
                }
 
                if (tdb_lock(tdb, tlock->hash, tlock->lock_rw) == -1)
-                       return -1;
+                       return TDB_NEXT_LOCK_ERR;
 
                /* No previous record?  Start at top of chain. */
                if (!tlock->off) {
@@ -99,6 +102,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
 
                        /* Detect infinite loops. From "Shlomi Yaakobovich" <Shlomi@exanet.com>. */
                        if (tlock->off == rec->next) {
+                               tdb->ecode = TDB_ERR_CORRUPT;
                                TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: loop detected.\n"));
                                goto fail;
                        }
@@ -128,7 +132,7 @@ static int tdb_next_lock(struct tdb_context *tdb, struct tdb_traverse_lock *tloc
        tlock->off = 0;
        if (tdb_unlock(tdb, tlock->hash, tlock->lock_rw) != 0)
                TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_next_lock: On error unlock failed!\n"));
-       return -1;
+       return TDB_NEXT_LOCK_ERR;
 }
 
 /* traverse the entire database - calling fn(tdb, key, data) on each element.
@@ -142,7 +146,8 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
 {
        TDB_DATA key, dbuf;
        struct list_struct rec;
-       int ret, count = 0;
+       int ret = 0, count = 0;
+       tdb_off_t off;
 
        /* This was in the initializaton, above, but the IRIX compiler
         * did not like it.  crh
@@ -153,7 +158,11 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
        tdb->travlocks.next = tl;
 
        /* tdb_next_lock places locks on the record returned, and its chain */
-       while ((ret = tdb_next_lock(tdb, tl, &rec)) > 0) {
+       while ((off = tdb_next_lock(tdb, tl, &rec)) != 0) {
+               if (off == TDB_NEXT_LOCK_ERR) {
+                       ret = -1;
+                       goto out;
+               }
                count++;
                /* now read the full record */
                key.dptr = tdb_alloc_read(tdb, tl->off + sizeof(rec), 
@@ -181,7 +190,6 @@ static int tdb_traverse_internal(struct tdb_context *tdb,
                if (fn && fn(tdb, key, dbuf, private_data)) {
                        /* They want us to terminate traversal */
                        tdb_trace_ret(tdb, "tdb_traverse_end", count);
-                       ret = count;
                        if (tdb_unlock_record(tdb, tl->off) != 0) {
                                TDB_LOG((tdb, TDB_DEBUG_FATAL, "tdb_traverse: unlock_record failed!\n"));;
                                ret = -1;
@@ -263,6 +271,7 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb)
 {
        TDB_DATA key;
        struct list_struct rec;
+       tdb_off_t off;
 
        /* release any old lock */
        if (tdb_unlock_record(tdb, tdb->travlocks.off) != 0)
@@ -271,7 +280,8 @@ TDB_DATA tdb_firstkey(struct tdb_context *tdb)
        tdb->travlocks.lock_rw = F_RDLCK;
 
        /* Grab first record: locks chain and returned record. */
-       if (tdb_next_lock(tdb, &tdb->travlocks, &rec) <= 0) {
+       off = tdb_next_lock(tdb, &tdb->travlocks, &rec);
+       if (off == 0 || off == TDB_NEXT_LOCK_ERR) {
                tdb_trace_retrec(tdb, "tdb_firstkey", tdb_null);
                return tdb_null;
        }
@@ -294,6 +304,7 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey)
        TDB_DATA key = tdb_null;
        struct list_struct rec;
        unsigned char *k = NULL;
+       tdb_off_t off;
 
        /* Is locked key the old key?  If so, traverse will be reliable. */
        if (tdb->travlocks.off) {
@@ -337,7 +348,8 @@ TDB_DATA tdb_nextkey(struct tdb_context *tdb, TDB_DATA oldkey)
 
        /* Grab next record: locks chain and returned record,
           unlocks old record */
-       if (tdb_next_lock(tdb, &tdb->travlocks, &rec) > 0) {
+       off = tdb_next_lock(tdb, &tdb->travlocks, &rec);
+       if (off != TDB_NEXT_LOCK_ERR && off != 0) {
                key.dsize = rec.key_len;
                key.dptr = tdb_alloc_read(tdb, tdb->travlocks.off+sizeof(rec),
                                          key.dsize);