tdb2: tdb_expand on empty database now tested.

[ccan] / ccan / tdb2 / io.c

diff --git a/ccan/tdb2/io.c b/ccan/tdb2/io.c
index 5910fc543d6dd64b3df56d2ba8376752bf25b6b8..cdee88aa1fd57321220d121a5bfe381332dc9269 100644 (file)
--- a/ccan/tdb2/io.c
+++ b/ccan/tdb2/io.c
@@ -125,11 +125,8 @@ static void *tdb_direct(struct tdb_context *tdb, tdb_off_t off, size_t len)
 /* Either make a copy into pad and return that, or return ptr into mmap. */
 /* Note: pad has to be a real object, so we can't get here if len
  * overflows size_t */
 /* Either make a copy into pad and return that, or return ptr into mmap. */
 /* Note: pad has to be a real object, so we can't get here if len
  * overflows size_t */

-/* FIXME: Transaction */

 void *tdb_get(struct tdb_context *tdb, tdb_off_t off, void *pad, size_t len)
 {
 void *tdb_get(struct tdb_context *tdb, tdb_off_t off, void *pad, size_t len)
 {

-       ssize_t r;
-

        if (likely(!(tdb->flags & TDB_CONVERT))) {
                void *ret = tdb_direct(tdb, off, len);
                if (ret)
        if (likely(!(tdb->flags & TDB_CONVERT))) {
                void *ret = tdb_direct(tdb, off, len);
                if (ret)


@@ -139,18 +136,8 @@ void *tdb_get(struct tdb_context *tdb, tdb_off_t off, void *pad, size_t len)
        if (unlikely(tdb_oob(tdb, off + len, false) == -1))
                return NULL;
 
        if (unlikely(tdb_oob(tdb, off + len, false) == -1))
                return NULL;
 

-       r = pread(tdb->fd, pad, len, off);
-       if (r != (ssize_t)len) {
-               /* Ensure ecode is set for log fn. */
-               tdb->ecode = TDB_ERR_IO;
-               tdb->log(tdb, TDB_DEBUG_FATAL, tdb->log_priv,
-                        "tdb_read failed at %llu "
-                        "len=%lld ret=%lld (%s) map_size=%lld\n",
-                        (long long)off, (long long)len,
-                        (long long)r, strerror(errno),
-                        (long long)tdb->map_size);
+       if (tdb->methods->read(tdb, off, pad, len) == -1)

                return NULL;
                return NULL;

-       }

        return tdb_convert(tdb, pad, len);
 }
 
        return tdb_convert(tdb, pad, len);
 }
 


@@ -241,7 +228,7 @@ int zero_out(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len)
                return 0;
        } else {
                char buf[8192] = { 0 };
                return 0;
        } else {
                char buf[8192] = { 0 };

-               return fill(tdb, buf, sizeof(buf), len, off);
+               return fill(tdb, buf, sizeof(buf), off, len);

        }
 }
 
        }
 }
 


@@ -249,7 +236,7 @@ tdb_off_t tdb_read_off(struct tdb_context *tdb, tdb_off_t off)
 {
        tdb_off_t pad, *ret;
 
 {
        tdb_off_t pad, *ret;
 

-       ret = tdb_get(tdb, off, &pad, sizeof(ret));
+       ret = tdb_get(tdb, off, &pad, sizeof(pad));

        if (!ret) {
                return TDB_OFF_ERR;
        }
        if (!ret) {
                return TDB_OFF_ERR;
        }


@@ -260,7 +247,7 @@ tdb_off_t tdb_read_off(struct tdb_context *tdb, tdb_off_t off)
 bool tdb_pwrite_all(int fd, const void *buf, size_t len, tdb_off_t off)
 {
        while (len) {
 bool tdb_pwrite_all(int fd, const void *buf, size_t len, tdb_off_t off)
 {
        while (len) {

-               size_t ret;
+               ssize_t ret;

                ret = pwrite(fd, buf, len, off);
                if (ret < 0)
                        return false;
                ret = pwrite(fd, buf, len, off);
                if (ret < 0)
                        return false;


@@ -268,13 +255,51 @@ bool tdb_pwrite_all(int fd, const void *buf, size_t len, tdb_off_t off)
                        errno = ENOSPC;
                        return false;
                }
                        errno = ENOSPC;
                        return false;
                }

-               buf += ret;
+               buf = (char *)buf + ret;
+               off += ret;
+               len -= ret;
+       }
+       return true;
+}
+
+/* Even on files, we can get partial reads due to signals. */
+bool tdb_pread_all(int fd, void *buf, size_t len, tdb_off_t off)
+{
+       while (len) {
+               ssize_t ret;
+               ret = pread(fd, buf, len, off);
+               if (ret < 0)
+                       return false;
+               if (ret == 0) {
+                       /* ETOOSHORT? */
+                       errno = EWOULDBLOCK;
+                       return false;
+               }
+               buf = (char *)buf + ret;

                off += ret;
                len -= ret;
        }
        return true;
 }
 
                off += ret;
                len -= ret;
        }
        return true;
 }
 

+bool tdb_read_all(int fd, void *buf, size_t len)
+{
+       while (len) {
+               ssize_t ret;
+               ret = read(fd, buf, len);
+               if (ret < 0)
+                       return false;
+               if (ret == 0) {
+                       /* ETOOSHORT? */
+                       errno = EWOULDBLOCK;
+                       return false;
+               }
+               buf = (char *)buf + ret;
+               len -= ret;
+       }
+       return true;
+}
+

 /* write a lump of data at a specified offset */
 static int tdb_write(struct tdb_context *tdb, tdb_off_t off, 
                     const void *buf, tdb_len_t len)
 /* write a lump of data at a specified offset */
 static int tdb_write(struct tdb_context *tdb, tdb_off_t off, 
                     const void *buf, tdb_len_t len)


@@ -316,15 +341,14 @@ static int tdb_read(struct tdb_context *tdb, tdb_off_t off, void *buf,
        if (tdb->map_ptr) {
                memcpy(buf, off + (char *)tdb->map_ptr, len);
        } else {
        if (tdb->map_ptr) {
                memcpy(buf, off + (char *)tdb->map_ptr, len);
        } else {

-               ssize_t ret = pread(tdb->fd, buf, len, off);
-               if (ret != (ssize_t)len) {
+               if (!tdb_pread_all(tdb->fd, buf, len, off)) {

                        /* Ensure ecode is set for log fn. */
                        tdb->ecode = TDB_ERR_IO;
                        tdb->log(tdb, TDB_DEBUG_FATAL, tdb->log_priv,
                                 "tdb_read failed at %lld "
                        /* Ensure ecode is set for log fn. */
                        tdb->ecode = TDB_ERR_IO;
                        tdb->log(tdb, TDB_DEBUG_FATAL, tdb->log_priv,
                                 "tdb_read failed at %lld "

-                                "len=%lld ret=%lld (%s) map_size=%lld\n",
+                                "len=%lld (%s) map_size=%lld\n",

                                 (long long)off, (long long)len,
                                 (long long)off, (long long)len,

-                                (long long)ret, strerror(errno),
+                                strerror(errno),

                                 (long long)tdb->map_size);
                        return -1;
                }
                                 (long long)tdb->map_size);
                        return -1;
                }


@@ -376,17 +400,17 @@ uint64_t hash_record(struct tdb_context *tdb, tdb_off_t off)
        void *key;
        uint64_t klen, hash;
 
        void *key;
        uint64_t klen, hash;
 

-       r = tdb_get(tdb, off, &pad, sizeof(*r));
+       r = tdb_get(tdb, off, &pad, sizeof(pad));

        if (!r)
                /* FIXME */
                return 0;
 
        klen = rec_key_length(r);
        if (!r)
                /* FIXME */
                return 0;
 
        klen = rec_key_length(r);

-       key = tdb_direct(tdb, off + sizeof(*r), klen);
+       key = tdb_direct(tdb, off + sizeof(pad), klen);

        if (likely(key))
                return tdb_hash(tdb, key, klen);
 
        if (likely(key))
                return tdb_hash(tdb, key, klen);
 

-       key = tdb_alloc_read(tdb, off + sizeof(*r), klen);
+       key = tdb_alloc_read(tdb, off + sizeof(pad), klen);

        if (unlikely(!key))
                return 0;
        hash = tdb_hash(tdb, key, klen);
        if (unlikely(!key))
                return 0;
        hash = tdb_hash(tdb, key, klen);


@@ -421,8 +445,7 @@ int tdb_parse_data(struct tdb_context *tdb, TDB_DATA key,
 
 /* expand a file.  we prefer to use ftruncate, as that is what posix
   says to use for mmap expansion */
 
 /* expand a file.  we prefer to use ftruncate, as that is what posix
   says to use for mmap expansion */

-static int tdb_expand_file(struct tdb_context *tdb,
-                          tdb_len_t size, tdb_len_t addition)
+static int tdb_expand_file(struct tdb_context *tdb, tdb_len_t addition)

 {
        char buf[8192];
 
 {
        char buf[8192];
 


@@ -431,15 +454,33 @@ static int tdb_expand_file(struct tdb_context *tdb,
                return -1;
        }
 
                return -1;
        }
 

-       /* If this fails, we try to fill anyway. */
-       if (ftruncate(tdb->fd, size+addition))
-               ;
+       if (tdb->flags & TDB_INTERNAL) {
+               char *new = realloc(tdb->map_ptr, tdb->map_size + addition);
+               if (!new) {
+                       tdb->ecode = TDB_ERR_OOM;
+                       return -1;
+               }
+               tdb->map_ptr = new;
+               tdb->map_size += addition;
+       } else {
+               /* Unmap before trying to write; old TDB claimed OpenBSD had
+                * problem with this otherwise. */
+               tdb_munmap(tdb);
+
+               /* If this fails, we try to fill anyway. */
+               if (ftruncate(tdb->fd, tdb->map_size + addition))
+                       ;

 
 

-       /* now fill the file with something. This ensures that the
-          file isn't sparse, which would be very bad if we ran out of
-          disk. This must be done with write, not via mmap */
-       memset(buf, 0x43, sizeof(buf));
-       return fill(tdb, buf, sizeof(buf), addition, size);
+               /* now fill the file with something. This ensures that the
+                  file isn't sparse, which would be very bad if we ran out of
+                  disk. This must be done with write, not via mmap */
+               memset(buf, 0x43, sizeof(buf));
+               if (fill(tdb, buf, sizeof(buf), tdb->map_size, addition) == -1)
+                       return -1;
+               tdb->map_size += addition;
+               tdb_mmap(tdb);
+       }
+       return 0;

 }
 
 const void *tdb_access_read(struct tdb_context *tdb,
 }
 
 const void *tdb_access_read(struct tdb_context *tdb,


@@ -534,84 +575,6 @@ static void tdb_next_hash_chain(struct tdb_context *tdb, uint32_t *chain)
        (*chain) = h;
 }
 
        (*chain) = h;
 }
 

-
-/* expand the database by expanding the underlying file and doing the
-   mmap again if necessary */
-int tdb_expand(struct tdb_context *tdb)
-{
-       struct tdb_record rec;
-       tdb_off_t offset, new_size;     
-
-       /* We have to lock every hash bucket and every free list. */
-       do {
-               
-
-       if (tdb_lock(tdb, -1, F_WRLCK) == -1) {
-               TDB_LOG((tdb, TDB_DEBUG_ERROR, "lock failed in tdb_expand\n"));
-               return -1;
-       }
-
-       /* must know about any previous expansions by another process */
-       tdb->methods->tdb_oob(tdb, tdb->map_size + 1, 1);
-
-       /* always make room for at least 100 more records, and at
-           least 25% more space. Round the database up to a multiple
-           of the page size */
-       new_size = MAX(tdb->map_size + size*100, tdb->map_size * 1.25);
-       size = TDB_ALIGN(new_size, tdb->page_size) - tdb->map_size;
-
-       if (!(tdb->flags & TDB_INTERNAL))
-               tdb_munmap(tdb);
-
-       /*
-        * We must ensure the file is unmapped before doing this
-        * to ensure consistency with systems like OpenBSD where
-        * writes and mmaps are not consistent.
-        */
-
-       /* expand the file itself */
-       if (!(tdb->flags & TDB_INTERNAL)) {
-               if (tdb->methods->tdb_expand_file(tdb, tdb->map_size, size) != 0)
-                       goto fail;
-       }
-
-       tdb->map_size += size;
-
-       if (tdb->flags & TDB_INTERNAL) {
-               char *new_map_ptr = (char *)realloc(tdb->map_ptr,
-                                                   tdb->map_size);
-               if (!new_map_ptr) {
-                       tdb->map_size -= size;
-                       goto fail;
-               }
-               tdb->map_ptr = new_map_ptr;
-       } else {
-               /*
-                * We must ensure the file is remapped before adding the space
-                * to ensure consistency with systems like OpenBSD where
-                * writes and mmaps are not consistent.
-                */
-
-               /* We're ok if the mmap fails as we'll fallback to read/write */
-               tdb_mmap(tdb);
-       }
-
-       /* form a new freelist record */
-       memset(&rec,'\0',sizeof(rec));
-       rec.rec_len = size - sizeof(rec);
-
-       /* link it into the free list */
-       offset = tdb->map_size - size;
-       if (tdb_free(tdb, offset, &rec) == -1)
-               goto fail;
-
-       tdb_unlock(tdb, -1, F_WRLCK);
-       return 0;
- fail:
-       tdb_unlock(tdb, -1, F_WRLCK);
-       return -1;
-}
-

 /* read/write a tdb_off_t */
 int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
 {
 /* read/write a tdb_off_t */
 int tdb_ofs_read(struct tdb_context *tdb, tdb_off_t offset, tdb_off_t *d)
 {