X-Git-Url: http://git.ozlabs.org/?p=ccan;a=blobdiff_plain;f=ccan%2Ftdb2%2Ftdb1_io.c;h=f3d139d0434a97fc7c9dbb6a4deea6cb6d2cd43e;hp=212e94ae14488a6dec32c8f3fc8f9b36a688609a;hb=380372e733416c2b348d5307f536d0a0807e95df;hpb=19e6c1a250ade1e7204ada17163294855585e825 diff --git a/ccan/tdb2/tdb1_io.c b/ccan/tdb2/tdb1_io.c index 212e94ae..f3d139d0 100644 --- a/ccan/tdb2/tdb1_io.c +++ b/ccan/tdb2/tdb1_io.c @@ -36,87 +36,86 @@ if necessary note that "len" is the minimum length needed for the db */ -static int tdb1_oob(struct tdb1_context *tdb, tdb1_off_t len, int probe) +static int tdb1_oob(struct tdb_context *tdb, tdb1_off_t len, int probe) { struct stat st; - if (len <= tdb->map_size) + if (len <= tdb->file->map_size) return 0; - if (tdb->flags & TDB1_INTERNAL) { + if (tdb->flags & TDB_INTERNAL) { if (!probe) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL,"tdb1_oob len %d beyond internal malloc size %d\n", - (int)len, (int)tdb->map_size)); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "tdb1_oob len %d beyond internal malloc size %d", + (int)len, (int)tdb->file->map_size); } return -1; } - if (fstat(tdb->fd, &st) == -1) { - tdb->ecode = TDB1_ERR_IO; + if (fstat(tdb->file->fd, &st) == -1) { + tdb->last_error = TDB_ERR_IO; return -1; } if (st.st_size < (size_t)len) { if (!probe) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL,"tdb1_oob len %d beyond eof at %d\n", - (int)len, (int)st.st_size)); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "tdb1_oob len %d beyond eof at %d", + (int)len, (int)st.st_size); } return -1; } /* Unmap, update size, remap */ if (tdb1_munmap(tdb) == -1) { - tdb->ecode = TDB1_ERR_IO; + tdb->last_error = TDB_ERR_IO; return -1; } - tdb->map_size = st.st_size; + tdb->file->map_size = st.st_size; tdb1_mmap(tdb); return 0; } /* write a lump of data at a specified offset */ -static int tdb1_write(struct tdb1_context *tdb, tdb1_off_t off, +static int tdb1_write(struct tdb_context *tdb, tdb1_off_t off, const void *buf, tdb1_len_t len) { if (len == 0) { return 0; } - if (tdb->read_only || tdb->traverse_read) { - tdb->ecode = TDB1_ERR_RDONLY; + if ((tdb->flags & TDB_RDONLY) || tdb->tdb1.traverse_read) { + tdb->last_error = TDB_ERR_RDONLY; return -1; } - if (tdb->methods->tdb1_oob(tdb, off + len, 0) != 0) + if (tdb->tdb1.io->tdb1_oob(tdb, off + len, 0) != 0) return -1; - if (tdb->map_ptr) { - memcpy(off + (char *)tdb->map_ptr, buf, len); + if (tdb->file->map_ptr) { + memcpy(off + (char *)tdb->file->map_ptr, buf, len); } else { - ssize_t written = pwrite(tdb->fd, buf, len, off); + ssize_t written = pwrite(tdb->file->fd, buf, len, off); if ((written != (ssize_t)len) && (written != -1)) { - /* try once more */ - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL, "tdb1_write: wrote only " - "%d of %d bytes at %d, trying once more\n", - (int)written, len, off)); - written = pwrite(tdb->fd, (const char *)buf+written, + tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING, + "tdb1_write: wrote only " + "%d of %d bytes at %d, trying once more", + (int)written, len, off); + written = pwrite(tdb->file->fd, + (const char *)buf+written, len-written, off+written); } if (written == -1) { /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL,"tdb1_write failed at %d " - "len=%d (%s)\n", off, len, strerror(errno))); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "tdb1_write failed at %d " + "len=%d (%s)", + off, len, strerror(errno)); return -1; } else if (written != (ssize_t)len) { - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL, "tdb1_write: failed to " - "write %d bytes at %d in two attempts\n", - len, off)); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "tdb1_write: failed to " + "write %d bytes at %d in two attempts", + len, off); return -1; } } @@ -134,24 +133,25 @@ void *tdb1_convert(void *buf, uint32_t size) /* read a lump of data at a specified offset, maybe convert */ -static int tdb1_read(struct tdb1_context *tdb, tdb1_off_t off, void *buf, +static int tdb1_read(struct tdb_context *tdb, tdb1_off_t off, void *buf, tdb1_len_t len, int cv) { - if (tdb->methods->tdb1_oob(tdb, off + len, 0) != 0) { + if (tdb->tdb1.io->tdb1_oob(tdb, off + len, 0) != 0) { return -1; } - if (tdb->map_ptr) { - memcpy(buf, off + (char *)tdb->map_ptr, len); + if (tdb->file->map_ptr) { + memcpy(buf, off + (char *)tdb->file->map_ptr, len); } else { - ssize_t ret = pread(tdb->fd, buf, len, off); + ssize_t ret = pread(tdb->file->fd, buf, len, off); if (ret != (ssize_t)len) { /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_IO; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL,"tdb1_read failed at %d " - "len=%d ret=%d (%s) map_size=%d\n", - (int)off, (int)len, (int)ret, strerror(errno), - (int)tdb->map_size)); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "tdb1_read failed at %d " + "len=%d ret=%d (%s) map_size=%d", + (int)off, (int)len, (int)ret, + strerror(errno), + (int)tdb->file->map_size); return -1; } } @@ -167,18 +167,18 @@ static int tdb1_read(struct tdb1_context *tdb, tdb1_off_t off, void *buf, do an unlocked scan of the hash table heads to find the next non-zero head. The value will then be confirmed with the lock held */ -static void tdb1_next_hash_chain(struct tdb1_context *tdb, uint32_t *chain) +static void tdb1_next_hash_chain(struct tdb_context *tdb, uint32_t *chain) { uint32_t h = *chain; - if (tdb->map_ptr) { - for (;h < tdb->header.hash_size;h++) { - if (0 != *(uint32_t *)(TDB1_HASH_TOP(h) + (unsigned char *)tdb->map_ptr)) { + if (tdb->file->map_ptr) { + for (;h < tdb->tdb1.header.hash_size;h++) { + if (0 != *(uint32_t *)(TDB1_HASH_TOP(h) + (unsigned char *)tdb->file->map_ptr)) { break; } } } else { uint32_t off=0; - for (;h < tdb->header.hash_size;h++) { + for (;h < tdb->tdb1.header.hash_size;h++) { if (tdb1_ofs_read(tdb, TDB1_HASH_TOP(h), &off) != 0 || off != 0) { break; } @@ -188,77 +188,88 @@ static void tdb1_next_hash_chain(struct tdb1_context *tdb, uint32_t *chain) } -int tdb1_munmap(struct tdb1_context *tdb) +int tdb1_munmap(struct tdb_context *tdb) { - if (tdb->flags & TDB1_INTERNAL) + if (tdb->flags & TDB_INTERNAL) return 0; #if HAVE_MMAP - if (tdb->map_ptr) { + if (tdb->file->map_ptr) { int ret; - ret = munmap(tdb->map_ptr, tdb->map_size); + ret = munmap(tdb->file->map_ptr, tdb->file->map_size); if (ret != 0) return ret; } #endif - tdb->map_ptr = NULL; + tdb->file->map_ptr = NULL; return 0; } -void tdb1_mmap(struct tdb1_context *tdb) +void tdb1_mmap(struct tdb_context *tdb) { - if (tdb->flags & TDB1_INTERNAL) + if (tdb->flags & TDB_INTERNAL) return; #if HAVE_MMAP - if (!(tdb->flags & TDB1_NOMMAP)) { - tdb->map_ptr = mmap(NULL, tdb->map_size, - PROT_READ|(tdb->read_only? 0:PROT_WRITE), - MAP_SHARED|MAP_FILE, tdb->fd, 0); + if (!(tdb->flags & TDB_NOMMAP)) { + int mmap_flags; + if ((tdb->open_flags & O_ACCMODE) == O_RDONLY) + mmap_flags = PROT_READ; + else + mmap_flags = PROT_READ | PROT_WRITE; + + tdb->file->map_ptr = mmap(NULL, tdb->file->map_size, + mmap_flags, + MAP_SHARED|MAP_FILE, tdb->file->fd, 0); /* * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!! */ - if (tdb->map_ptr == MAP_FAILED) { - tdb->map_ptr = NULL; - TDB1_LOG((tdb, TDB1_DEBUG_WARNING, "tdb1_mmap failed for size %d (%s)\n", - tdb->map_size, strerror(errno))); + if (tdb->file->map_ptr == MAP_FAILED) { + tdb->file->map_ptr = NULL; + tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING, + "tdb1_mmap failed for size %d (%s)", + tdb->file->map_size, strerror(errno)); } } else { - tdb->map_ptr = NULL; + tdb->file->map_ptr = NULL; } #else - tdb->map_ptr = NULL; + tdb->file->map_ptr = NULL; #endif } /* expand a file. we prefer to use ftruncate, as that is what posix says to use for mmap expansion */ -static int tdb1_expand_file(struct tdb1_context *tdb, tdb1_off_t size, tdb1_off_t addition) +static int tdb1_expand_file(struct tdb_context *tdb, tdb1_off_t size, tdb1_off_t addition) { char buf[8192]; - if (tdb->read_only || tdb->traverse_read) { - tdb->ecode = TDB1_ERR_RDONLY; + if ((tdb->flags & TDB_RDONLY) || tdb->tdb1.traverse_read) { + tdb->last_error = TDB_ERR_RDONLY; return -1; } - if (ftruncate(tdb->fd, size+addition) == -1) { + if (ftruncate(tdb->file->fd, size+addition) == -1) { char b = 0; - ssize_t written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + ssize_t written = pwrite(tdb->file->fd, &b, 1, + (size+addition) - 1); if (written == 0) { /* try once more, potentially revealing errno */ - written = pwrite(tdb->fd, &b, 1, (size+addition) - 1); + written = pwrite(tdb->file->fd, &b, 1, + (size+addition) - 1); } if (written == 0) { /* again - give up, guessing errno */ errno = ENOSPC; } if (written != 1) { - TDB1_LOG((tdb, TDB1_DEBUG_FATAL, "expand_file to %d failed (%s)\n", - size+addition, strerror(errno))); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "expand_file to %d failed (%s)", + size+addition, + strerror(errno)); return -1; } } @@ -269,71 +280,76 @@ static int tdb1_expand_file(struct tdb1_context *tdb, tdb1_off_t size, tdb1_off_ memset(buf, TDB1_PAD_BYTE, sizeof(buf)); while (addition) { size_t n = addition>sizeof(buf)?sizeof(buf):addition; - ssize_t written = pwrite(tdb->fd, buf, n, size); + ssize_t written = pwrite(tdb->file->fd, buf, n, size); if (written == 0) { /* prevent infinite loops: try _once_ more */ - written = pwrite(tdb->fd, buf, n, size); + written = pwrite(tdb->file->fd, buf, n, size); } if (written == 0) { /* give up, trying to provide a useful errno */ - TDB1_LOG((tdb, TDB1_DEBUG_FATAL, "expand_file write " - "returned 0 twice: giving up!\n")); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "expand_file write " + "returned 0 twice: giving up!"); errno = ENOSPC; return -1; } else if (written == -1) { - TDB1_LOG((tdb, TDB1_DEBUG_FATAL, "expand_file write of " - "%d bytes failed (%s)\n", (int)n, - strerror(errno))); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR, + "expand_file write of " + "%d bytes failed (%s)", (int)n, + strerror(errno)); return -1; } else if (written != n) { - TDB1_LOG((tdb, TDB1_DEBUG_WARNING, "expand_file: wrote " - "only %d of %d bytes - retrying\n", (int)written, - (int)n)); + tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING, + "expand_file: wrote " + "only %d of %d bytes - retrying", + (int)written, (int)n); } addition -= written; size += written; } + tdb->stats.expands++; return 0; } /* expand the database at least size bytes by expanding the underlying file and doing the mmap again if necessary */ -int tdb1_expand(struct tdb1_context *tdb, tdb1_off_t size) +int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size) { struct tdb1_record rec; tdb1_off_t offset, new_size, top_size, map_size; if (tdb1_lock(tdb, -1, F_WRLCK) == -1) { - TDB1_LOG((tdb, TDB1_DEBUG_ERROR, "lock failed in tdb1_expand\n")); + tdb_logerr(tdb, tdb->last_error, TDB_LOG_ERROR, + "lock failed in tdb1_expand"); return -1; } /* must know about any previous expansions by another process */ - tdb->methods->tdb1_oob(tdb, tdb->map_size + 1, 1); + tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size + 1, 1); /* limit size in order to avoid using up huge amounts of memory for * in memory tdbs if an oddball huge record creeps in */ if (size > 100 * 1024) { - top_size = tdb->map_size + size * 2; + top_size = tdb->file->map_size + size * 2; } else { - top_size = tdb->map_size + size * 100; + top_size = tdb->file->map_size + size * 100; } /* always make room for at least top_size more records, and at least 25% more space. if the DB is smaller than 100MiB, otherwise grow it by 10% only. */ - if (tdb->map_size > 100 * 1024 * 1024) { - map_size = tdb->map_size * 1.10; + if (tdb->file->map_size > 100 * 1024 * 1024) { + map_size = tdb->file->map_size * 1.10; } else { - map_size = tdb->map_size * 1.25; + map_size = tdb->file->map_size * 1.25; } /* Round the database up to a multiple of the page size */ new_size = MAX(top_size, map_size); - size = TDB1_ALIGN(new_size, tdb->page_size) - tdb->map_size; + size = TDB1_ALIGN(new_size, tdb->tdb1.page_size) - tdb->file->map_size; - if (!(tdb->flags & TDB1_INTERNAL)) + if (!(tdb->flags & TDB_INTERNAL)) tdb1_munmap(tdb); /* @@ -343,21 +359,24 @@ int tdb1_expand(struct tdb1_context *tdb, tdb1_off_t size) */ /* expand the file itself */ - if (!(tdb->flags & TDB1_INTERNAL)) { - if (tdb->methods->tdb1_expand_file(tdb, tdb->map_size, size) != 0) + if (!(tdb->flags & TDB_INTERNAL)) { + if (tdb->tdb1.io->tdb1_expand_file(tdb, tdb->file->map_size, size) != 0) goto fail; } - tdb->map_size += size; + tdb->file->map_size += size; - if (tdb->flags & TDB1_INTERNAL) { - char *new_map_ptr = (char *)realloc(tdb->map_ptr, - tdb->map_size); + if (tdb->flags & TDB_INTERNAL) { + char *new_map_ptr = (char *)realloc(tdb->file->map_ptr, + tdb->file->map_size); if (!new_map_ptr) { - tdb->map_size -= size; + tdb->last_error = tdb_logerr(tdb, TDB_ERR_OOM, + TDB_LOG_ERROR, + "tdb1_expand: no memory"); + tdb->file->map_size -= size; goto fail; } - tdb->map_ptr = new_map_ptr; + tdb->file->map_ptr = new_map_ptr; } else { /* * We must ensure the file is remapped before adding the space @@ -374,7 +393,7 @@ int tdb1_expand(struct tdb1_context *tdb, tdb1_off_t size) rec.rec_len = size - sizeof(rec); /* link it into the free list */ - offset = tdb->map_size - size; + offset = tdb->file->map_size - size; if (tdb1_free(tdb, offset, &rec) == -1) goto fail; @@ -386,33 +405,33 @@ int tdb1_expand(struct tdb1_context *tdb, tdb1_off_t size) } /* read/write a tdb1_off_t */ -int tdb1_ofs_read(struct tdb1_context *tdb, tdb1_off_t offset, tdb1_off_t *d) +int tdb1_ofs_read(struct tdb_context *tdb, tdb1_off_t offset, tdb1_off_t *d) { - return tdb->methods->tdb1_read(tdb, offset, (char*)d, sizeof(*d), TDB1_DOCONV()); + return tdb->tdb1.io->tdb1_read(tdb, offset, (char*)d, sizeof(*d), TDB1_DOCONV()); } -int tdb1_ofs_write(struct tdb1_context *tdb, tdb1_off_t offset, tdb1_off_t *d) +int tdb1_ofs_write(struct tdb_context *tdb, tdb1_off_t offset, tdb1_off_t *d) { tdb1_off_t off = *d; - return tdb->methods->tdb1_write(tdb, offset, TDB1_CONV(off), sizeof(*d)); + return tdb->tdb1.io->tdb1_write(tdb, offset, TDB1_CONV(off), sizeof(*d)); } /* read a lump of data, allocating the space for it */ -unsigned char *tdb1_alloc_read(struct tdb1_context *tdb, tdb1_off_t offset, tdb1_len_t len) +unsigned char *tdb1_alloc_read(struct tdb_context *tdb, tdb1_off_t offset, tdb1_len_t len) { unsigned char *buf; /* some systems don't like zero length malloc */ if (!(buf = (unsigned char *)malloc(len ? len : 1))) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_OOM; - TDB1_LOG((tdb, TDB1_DEBUG_ERROR,"tdb1_alloc_read malloc failed len=%d (%s)\n", - len, strerror(errno))); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR, + "tdb1_alloc_read malloc failed" + " len=%d (%s)", + len, strerror(errno)); return NULL; } - if (tdb->methods->tdb1_read(tdb, offset, buf, len, 0) == -1) { + if (tdb->tdb1.io->tdb1_read(tdb, offset, buf, len, 0) == -1) { SAFE_FREE(buf); return NULL; } @@ -420,32 +439,32 @@ unsigned char *tdb1_alloc_read(struct tdb1_context *tdb, tdb1_off_t offset, tdb1 } /* Give a piece of tdb data to a parser */ - -int tdb1_parse_data(struct tdb1_context *tdb, TDB1_DATA key, - tdb1_off_t offset, tdb1_len_t len, - int (*parser)(TDB1_DATA key, TDB1_DATA data, - void *private_data), - void *private_data) +enum TDB_ERROR tdb1_parse_data(struct tdb_context *tdb, TDB_DATA key, + tdb1_off_t offset, tdb1_len_t len, + enum TDB_ERROR (*parser)(TDB_DATA key, + TDB_DATA data, + void *private_data), + void *private_data) { - TDB1_DATA data; - int result; + TDB_DATA data; + enum TDB_ERROR result; data.dsize = len; - if ((tdb->transaction == NULL) && (tdb->map_ptr != NULL)) { + if ((tdb->tdb1.transaction == NULL) && (tdb->file->map_ptr != NULL)) { /* * Optimize by avoiding the malloc/memcpy/free, point the * parser directly at the mmap area. */ - if (tdb->methods->tdb1_oob(tdb, offset+len, 0) != 0) { - return -1; + if (tdb->tdb1.io->tdb1_oob(tdb, offset+len, 0) != 0) { + return tdb->last_error; } - data.dptr = offset + (unsigned char *)tdb->map_ptr; + data.dptr = offset + (unsigned char *)tdb->file->map_ptr; return parser(key, data, private_data); } if (!(data.dptr = tdb1_alloc_read(tdb, offset, len))) { - return -1; + return tdb->last_error; } result = parser(key, data, private_data); @@ -454,23 +473,23 @@ int tdb1_parse_data(struct tdb1_context *tdb, TDB1_DATA key, } /* read/write a record */ -int tdb1_rec_read(struct tdb1_context *tdb, tdb1_off_t offset, struct tdb1_record *rec) +int tdb1_rec_read(struct tdb_context *tdb, tdb1_off_t offset, struct tdb1_record *rec) { - if (tdb->methods->tdb1_read(tdb, offset, rec, sizeof(*rec),TDB1_DOCONV()) == -1) + if (tdb->tdb1.io->tdb1_read(tdb, offset, rec, sizeof(*rec),TDB1_DOCONV()) == -1) return -1; if (TDB1_BAD_MAGIC(rec)) { - /* Ensure ecode is set for log fn. */ - tdb->ecode = TDB1_ERR_CORRUPT; - TDB1_LOG((tdb, TDB1_DEBUG_FATAL,"tdb1_rec_read bad magic 0x%x at offset=%d\n", rec->magic, offset)); + tdb->last_error = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR, + "tdb1_rec_read bad magic 0x%x at offset=%d", + rec->magic, offset); return -1; } - return tdb->methods->tdb1_oob(tdb, rec->next+sizeof(*rec), 0); + return tdb->tdb1.io->tdb1_oob(tdb, rec->next+sizeof(*rec), 0); } -int tdb1_rec_write(struct tdb1_context *tdb, tdb1_off_t offset, struct tdb1_record *rec) +int tdb1_rec_write(struct tdb_context *tdb, tdb1_off_t offset, struct tdb1_record *rec) { struct tdb1_record r = *rec; - return tdb->methods->tdb1_write(tdb, offset, TDB1_CONV(r), sizeof(r)); + return tdb->tdb1.io->tdb1_write(tdb, offset, TDB1_CONV(r), sizeof(r)); } static const struct tdb1_methods io1_methods = { @@ -484,7 +503,14 @@ static const struct tdb1_methods io1_methods = { /* initialise the default methods table */ -void tdb1_io_init(struct tdb1_context *tdb) +void tdb1_io_init(struct tdb_context *tdb) +{ + tdb->tdb1.io = &io1_methods; +} + +enum TDB_ERROR tdb1_probe_length(struct tdb_context *tdb) { - tdb->methods = &io1_methods; + tdb->last_error = TDB_SUCCESS; + tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size + 1, true); + return tdb->last_error; }