git.ozlabs.org Git - ccan/blob - ccan/tdb2/tdb1_io.c

   1  /*
   2    Unix SMB/CIFS implementation.
   3
   4    trivial database library
   5
   6    Copyright (C) Andrew Tridgell              1999-2005
   7    Copyright (C) Paul `Rusty' Russell              2000
   8    Copyright (C) Jeremy Allison                    2000-2003
   9
  10      ** NOTE! The following LGPL license applies to the tdb
  11      ** library. This does NOT imply that all of Samba is released
  12      ** under the LGPL
  13
  14    This library is free software; you can redistribute it and/or
  15    modify it under the terms of the GNU Lesser General Public
  16    License as published by the Free Software Foundation; either
  17    version 3 of the License, or (at your option) any later version.
  18
  19    This library is distributed in the hope that it will be useful,
  20    but WITHOUT ANY WARRANTY; without even the implied warranty of
  21    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  22    Lesser General Public License for more details.
  23
  24    You should have received a copy of the GNU Lesser General Public
  25    License along with this library; if not, see <http://www.gnu.org/licenses/>.
  26 */
  27
  28
  29 #include "tdb1_private.h"
  30 #ifndef MAX
  31 #define MAX(a,b) ((a) > (b) ? (a) : (b))
  32 #endif
  33
  34 /* check for an out of bounds access - if it is out of bounds then
  35    see if the database has been expanded by someone else and expand
  36    if necessary
  37    note that "len" is the minimum length needed for the db
  38 */
  39 static int tdb1_oob(struct tdb_context *tdb, tdb1_off_t off, tdb1_len_t len,
  40                     int probe)
  41 {
  42         struct stat st;
  43         if (len + off < len) {
  44                 if (!probe) {
  45                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
  46                                                      "tdb1_oob off %d len %d wrap\n",
  47                                                      (int)off, (int)len);
  48                 }
  49                 return -1;
  50         }
  51
  52         if (off + len <= tdb->file->map_size)
  53                 return 0;
  54         if (tdb->flags & TDB_INTERNAL) {
  55                 if (!probe) {
  56                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
  57                                                      "tdb1_oob len %d beyond internal malloc size %u",
  58                                                      (int)(off + len), (int)tdb->file->map_size);
  59                 }
  60                 return -1;
  61         }
  62
  63         if (fstat(tdb->file->fd, &st) == -1) {
  64                 tdb->last_error = TDB_ERR_IO;
  65                 return -1;
  66         }
  67
  68         if (st.st_size < (size_t)off + len) {
  69                 if (!probe) {
  70                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
  71                                                      "tdb1_oob len %u beyond eof at %u",
  72                                                      (int)(off + len), (int)st.st_size);
  73                 }
  74                 return -1;
  75         }
  76
  77         /* Beware >4G files! */
  78         if ((tdb1_off_t)st.st_size != st.st_size) {
  79                 tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
  80                                              "tdb1_oob len %llu too large!\n",
  81                                              (long long)st.st_size);
  82                 return -1;
  83         }
  84
  85         /* Unmap, update size, remap */
  86         if (tdb1_munmap(tdb) == -1) {
  87                 tdb->last_error = TDB_ERR_IO;
  88                 return -1;
  89         }
  90         tdb->file->map_size = st.st_size;
  91         tdb1_mmap(tdb);
  92         return 0;
  93 }
  94
  95 /* write a lump of data at a specified offset */
  96 static int tdb1_write(struct tdb_context *tdb, tdb1_off_t off,
  97                      const void *buf, tdb1_len_t len)
  98 {
  99         if (len == 0) {
 100                 return 0;
 101         }
 102
 103         if ((tdb->flags & TDB_RDONLY) || tdb->tdb1.traverse_read) {
 104                 tdb->last_error = TDB_ERR_RDONLY;
 105                 return -1;
 106         }
 107
 108         if (tdb->tdb1.io->tdb1_oob(tdb, off, len, 0) != 0)
 109                 return -1;
 110
 111         if (tdb->file->map_ptr) {
 112                 memcpy(off + (char *)tdb->file->map_ptr, buf, len);
 113         } else {
 114                 ssize_t written = pwrite(tdb->file->fd, buf, len, off);
 115                 if ((written != (ssize_t)len) && (written != -1)) {
 116                         tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING,
 117                                    "tdb1_write: wrote only "
 118                                    "%d of %d bytes at %d, trying once more",
 119                                    (int)written, len, off);
 120                         written = pwrite(tdb->file->fd,
 121                                          (const char *)buf+written,
 122                                          len-written,
 123                                          off+written);
 124                 }
 125                 if (written == -1) {
 126                         /* Ensure ecode is set for log fn. */
 127                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 128                                                 "tdb1_write failed at %d "
 129                                                 "len=%d (%s)",
 130                                                 off, len, strerror(errno));
 131                         return -1;
 132                 } else if (written != (ssize_t)len) {
 133                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 134                                                 "tdb1_write: failed to "
 135                                                 "write %d bytes at %d in two attempts",
 136                                                 len, off);
 137                         return -1;
 138                 }
 139         }
 140         return 0;
 141 }
 142
 143 /* Endian conversion: we only ever deal with 4 byte quantities */
 144 void *tdb1_convert(void *buf, uint32_t size)
 145 {
 146         uint32_t i, *p = (uint32_t *)buf;
 147         for (i = 0; i < size / 4; i++)
 148                 p[i] = TDB1_BYTEREV(p[i]);
 149         return buf;
 150 }
 151
 152
 153 /* read a lump of data at a specified offset, maybe convert */
 154 static int tdb1_read(struct tdb_context *tdb, tdb1_off_t off, void *buf,
 155                     tdb1_len_t len, int cv)
 156 {
 157         if (tdb->tdb1.io->tdb1_oob(tdb, off, len, 0) != 0) {
 158                 return -1;
 159         }
 160
 161         if (tdb->file->map_ptr) {
 162                 memcpy(buf, off + (char *)tdb->file->map_ptr, len);
 163         } else {
 164                 ssize_t ret = pread(tdb->file->fd, buf, len, off);
 165                 if (ret != (ssize_t)len) {
 166                         /* Ensure ecode is set for log fn. */
 167                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 168                                                 "tdb1_read failed at %d "
 169                                                 "len=%d ret=%d (%s) map_size=%d",
 170                                                 (int)off, (int)len, (int)ret,
 171                                                 strerror(errno),
 172                                                 (int)tdb->file->map_size);
 173                         return -1;
 174                 }
 175         }
 176         if (cv) {
 177                 tdb1_convert(buf, len);
 178         }
 179         return 0;
 180 }
 181
 182
 183
 184 /*
 185   do an unlocked scan of the hash table heads to find the next non-zero head. The value
 186   will then be confirmed with the lock held
 187 */
 188 static void tdb1_next_hash_chain(struct tdb_context *tdb, uint32_t *chain)
 189 {
 190         uint32_t h = *chain;
 191         if (tdb->file->map_ptr) {
 192                 for (;h < tdb->tdb1.header.hash_size;h++) {
 193                         if (0 != *(uint32_t *)(TDB1_HASH_TOP(h) + (unsigned char *)tdb->file->map_ptr)) {
 194                                 break;
 195                         }
 196                 }
 197         } else {
 198                 uint32_t off=0;
 199                 for (;h < tdb->tdb1.header.hash_size;h++) {
 200                         if (tdb1_ofs_read(tdb, TDB1_HASH_TOP(h), &off) != 0 || off != 0) {
 201                                 break;
 202                         }
 203                 }
 204         }
 205         (*chain) = h;
 206 }
 207
 208
 209 int tdb1_munmap(struct tdb_context *tdb)
 210 {
 211         if (tdb->flags & TDB_INTERNAL)
 212                 return 0;
 213
 214 #if HAVE_MMAP
 215         if (tdb->file->map_ptr) {
 216                 int ret;
 217
 218                 ret = munmap(tdb->file->map_ptr, tdb->file->map_size);
 219                 if (ret != 0)
 220                         return ret;
 221         }
 222 #endif
 223         tdb->file->map_ptr = NULL;
 224         return 0;
 225 }
 226
 227 void tdb1_mmap(struct tdb_context *tdb)
 228 {
 229         if (tdb->flags & TDB_INTERNAL)
 230                 return;
 231
 232 #if HAVE_MMAP
 233         if (!(tdb->flags & TDB_NOMMAP)) {
 234                 int mmap_flags;
 235                 if ((tdb->open_flags & O_ACCMODE) == O_RDONLY)
 236                         mmap_flags = PROT_READ;
 237                 else
 238                         mmap_flags = PROT_READ | PROT_WRITE;
 239
 240                 tdb->file->map_ptr = mmap(NULL, tdb->file->map_size,
 241                                     mmap_flags,
 242                                     MAP_SHARED|MAP_FILE, tdb->file->fd, 0);
 243
 244                 /*
 245                  * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
 246                  */
 247
 248                 if (tdb->file->map_ptr == MAP_FAILED) {
 249                         tdb->file->map_ptr = NULL;
 250                         tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING,
 251                                    "tdb1_mmap failed for size %d (%s)",
 252                                    tdb->file->map_size, strerror(errno));
 253                 }
 254         } else {
 255                 tdb->file->map_ptr = NULL;
 256         }
 257 #else
 258         tdb->file->map_ptr = NULL;
 259 #endif
 260 }
 261
 262 /* expand a file.  we prefer to use ftruncate, as that is what posix
 263   says to use for mmap expansion */
 264 static int tdb1_expand_file(struct tdb_context *tdb, tdb1_off_t size, tdb1_off_t addition)
 265 {
 266         char buf[8192];
 267
 268         if ((tdb->flags & TDB_RDONLY) || tdb->tdb1.traverse_read) {
 269                 tdb->last_error = TDB_ERR_RDONLY;
 270                 return -1;
 271         }
 272
 273         if (ftruncate(tdb->file->fd, size+addition) == -1) {
 274                 char b = 0;
 275                 ssize_t written = pwrite(tdb->file->fd, &b, 1,
 276                                          (size+addition) - 1);
 277                 if (written == 0) {
 278                         /* try once more, potentially revealing errno */
 279                         written = pwrite(tdb->file->fd, &b, 1,
 280                                          (size+addition) - 1);
 281                 }
 282                 if (written == 0) {
 283                         /* again - give up, guessing errno */
 284                         errno = ENOSPC;
 285                 }
 286                 if (written != 1) {
 287                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 288                                                 "expand_file to %d failed (%s)",
 289                                                 size+addition,
 290                                                 strerror(errno));
 291                         return -1;
 292                 }
 293         }
 294
 295         /* now fill the file with something. This ensures that the
 296            file isn't sparse, which would be very bad if we ran out of
 297            disk. This must be done with write, not via mmap */
 298         memset(buf, TDB1_PAD_BYTE, sizeof(buf));
 299         while (addition) {
 300                 size_t n = addition>sizeof(buf)?sizeof(buf):addition;
 301                 ssize_t written = pwrite(tdb->file->fd, buf, n, size);
 302                 if (written == 0) {
 303                         /* prevent infinite loops: try _once_ more */
 304                         written = pwrite(tdb->file->fd, buf, n, size);
 305                 }
 306                 if (written == 0) {
 307                         /* give up, trying to provide a useful errno */
 308                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 309                                                 "expand_file write "
 310                                                 "returned 0 twice: giving up!");
 311                         errno = ENOSPC;
 312                         return -1;
 313                 } else if (written == -1) {
 314                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
 315                                                 "expand_file write of "
 316                                                 "%d bytes failed (%s)", (int)n,
 317                                                 strerror(errno));
 318                         return -1;
 319                 } else if (written != n) {
 320                         tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_WARNING,
 321                                    "expand_file: wrote "
 322                                    "only %d of %d bytes - retrying",
 323                                    (int)written, (int)n);
 324                 }
 325                 addition -= written;
 326                 size += written;
 327         }
 328         tdb->stats.expands++;
 329         return 0;
 330 }
 331
 332
 333 /* You need 'size', this tells you how much you should expand by. */
 334 tdb1_off_t tdb1_expand_adjust(tdb1_off_t map_size, tdb1_off_t size, int page_size)
 335 {
 336         tdb1_off_t new_size, top_size;
 337
 338         /* limit size in order to avoid using up huge amounts of memory for
 339          * in memory tdbs if an oddball huge record creeps in */
 340         if (size > 100 * 1024) {
 341                 top_size = map_size + size * 2;
 342         } else {
 343                 top_size = map_size + size * 100;
 344         }
 345
 346         /* always make room for at least top_size more records, and at
 347            least 25% more space. if the DB is smaller than 100MiB,
 348            otherwise grow it by 10% only. */
 349         if (map_size > 100 * 1024 * 1024) {
 350                 new_size = map_size * 1.10;
 351         } else {
 352                 new_size = map_size * 1.25;
 353         }
 354
 355         /* Round the database up to a multiple of the page size */
 356         new_size = MAX(top_size, new_size);
 357         return TDB1_ALIGN(new_size, page_size) - map_size;
 358 }
 359
 360 /* expand the database at least size bytes by expanding the underlying
 361    file and doing the mmap again if necessary */
 362 int tdb1_expand(struct tdb_context *tdb, tdb1_off_t size)
 363 {
 364         struct tdb1_record rec;
 365         tdb1_off_t offset;
 366
 367         if (tdb1_lock(tdb, -1, F_WRLCK) == -1) {
 368                 tdb_logerr(tdb, tdb->last_error, TDB_LOG_ERROR,
 369                            "lock failed in tdb1_expand");
 370                 return -1;
 371         }
 372
 373         /* must know about any previous expansions by another process */
 374         tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size, 1, 1);
 375
 376         size = tdb1_expand_adjust(tdb->file->map_size, size,
 377                                   tdb->tdb1.page_size);
 378
 379         if (!(tdb->flags & TDB_INTERNAL))
 380                 tdb1_munmap(tdb);
 381
 382         /*
 383          * We must ensure the file is unmapped before doing this
 384          * to ensure consistency with systems like OpenBSD where
 385          * writes and mmaps are not consistent.
 386          */
 387
 388         /* expand the file itself */
 389         if (!(tdb->flags & TDB_INTERNAL)) {
 390                 if (tdb->tdb1.io->tdb1_expand_file(tdb, tdb->file->map_size, size) != 0)
 391                         goto fail;
 392         }
 393
 394         tdb->file->map_size += size;
 395
 396         if (tdb->flags & TDB_INTERNAL) {
 397                 char *new_map_ptr = (char *)realloc(tdb->file->map_ptr,
 398                                                     tdb->file->map_size);
 399                 if (!new_map_ptr) {
 400                         tdb->last_error = tdb_logerr(tdb, TDB_ERR_OOM,
 401                                                      TDB_LOG_ERROR,
 402                                                      "tdb1_expand: no memory");
 403                         tdb->file->map_size -= size;
 404                         goto fail;
 405                 }
 406                 tdb->file->map_ptr = new_map_ptr;
 407         } else {
 408                 /*
 409                  * We must ensure the file is remapped before adding the space
 410                  * to ensure consistency with systems like OpenBSD where
 411                  * writes and mmaps are not consistent.
 412                  */
 413
 414                 /* We're ok if the mmap fails as we'll fallback to read/write */
 415                 tdb1_mmap(tdb);
 416         }
 417
 418         /* form a new freelist record */
 419         memset(&rec,'\0',sizeof(rec));
 420         rec.rec_len = size - sizeof(rec);
 421
 422         /* link it into the free list */
 423         offset = tdb->file->map_size - size;
 424         if (tdb1_free(tdb, offset, &rec) == -1)
 425                 goto fail;
 426
 427         tdb1_unlock(tdb, -1, F_WRLCK);
 428         return 0;
 429  fail:
 430         tdb1_unlock(tdb, -1, F_WRLCK);
 431         return -1;
 432 }
 433
 434 /* read/write a tdb1_off_t */
 435 int tdb1_ofs_read(struct tdb_context *tdb, tdb1_off_t offset, tdb1_off_t *d)
 436 {
 437         return tdb->tdb1.io->tdb1_read(tdb, offset, (char*)d, sizeof(*d), TDB1_DOCONV());
 438 }
 439
 440 int tdb1_ofs_write(struct tdb_context *tdb, tdb1_off_t offset, tdb1_off_t *d)
 441 {
 442         tdb1_off_t off = *d;
 443         return tdb->tdb1.io->tdb1_write(tdb, offset, TDB1_CONV(off), sizeof(*d));
 444 }
 445
 446
 447 /* read a lump of data, allocating the space for it */
 448 unsigned char *tdb1_alloc_read(struct tdb_context *tdb, tdb1_off_t offset, tdb1_len_t len)
 449 {
 450         unsigned char *buf;
 451
 452         /* some systems don't like zero length malloc */
 453
 454         if (!(buf = (unsigned char *)malloc(len ? len : 1))) {
 455                 tdb->last_error = tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
 456                                              "tdb1_alloc_read malloc failed"
 457                                              " len=%d (%s)",
 458                                              len, strerror(errno));
 459                 return NULL;
 460         }
 461         if (tdb->tdb1.io->tdb1_read(tdb, offset, buf, len, 0) == -1) {
 462                 SAFE_FREE(buf);
 463                 return NULL;
 464         }
 465         return buf;
 466 }
 467
 468 /* Give a piece of tdb data to a parser */
 469 enum TDB_ERROR tdb1_parse_data(struct tdb_context *tdb, TDB_DATA key,
 470                                tdb1_off_t offset, tdb1_len_t len,
 471                                enum TDB_ERROR (*parser)(TDB_DATA key,
 472                                                         TDB_DATA data,
 473                                                         void *private_data),
 474                                void *private_data)
 475 {
 476         TDB_DATA data;
 477         enum TDB_ERROR result;
 478
 479         data.dsize = len;
 480
 481         if ((tdb->tdb1.transaction == NULL) && (tdb->file->map_ptr != NULL)) {
 482                 /*
 483                  * Optimize by avoiding the malloc/memcpy/free, point the
 484                  * parser directly at the mmap area.
 485                  */
 486                 if (tdb->tdb1.io->tdb1_oob(tdb, offset, len, 0) != 0) {
 487                         return tdb->last_error;
 488                 }
 489                 data.dptr = offset + (unsigned char *)tdb->file->map_ptr;
 490                 return parser(key, data, private_data);
 491         }
 492
 493         if (!(data.dptr = tdb1_alloc_read(tdb, offset, len))) {
 494                 return tdb->last_error;
 495         }
 496
 497         result = parser(key, data, private_data);
 498         free(data.dptr);
 499         return result;
 500 }
 501
 502 /* read/write a record */
 503 int tdb1_rec_read(struct tdb_context *tdb, tdb1_off_t offset, struct tdb1_record *rec)
 504 {
 505         if (tdb->tdb1.io->tdb1_read(tdb, offset, rec, sizeof(*rec),TDB1_DOCONV()) == -1)
 506                 return -1;
 507         if (TDB1_BAD_MAGIC(rec)) {
 508                 tdb->last_error = tdb_logerr(tdb, TDB_ERR_CORRUPT, TDB_LOG_ERROR,
 509                                         "tdb1_rec_read bad magic 0x%x at offset=%d",
 510                                         rec->magic, offset);
 511                 return -1;
 512         }
 513         return tdb->tdb1.io->tdb1_oob(tdb, rec->next, sizeof(*rec), 0);
 514 }
 515
 516 int tdb1_rec_write(struct tdb_context *tdb, tdb1_off_t offset, struct tdb1_record *rec)
 517 {
 518         struct tdb1_record r = *rec;
 519         return tdb->tdb1.io->tdb1_write(tdb, offset, TDB1_CONV(r), sizeof(r));
 520 }
 521
 522 static const struct tdb1_methods io1_methods = {
 523         tdb1_read,
 524         tdb1_write,
 525         tdb1_next_hash_chain,
 526         tdb1_oob,
 527         tdb1_expand_file,
 528 };
 529
 530 /*
 531   initialise the default methods table
 532 */
 533 void tdb1_io_init(struct tdb_context *tdb)
 534 {
 535         tdb->tdb1.io = &io1_methods;
 536 }
 537
 538 enum TDB_ERROR tdb1_probe_length(struct tdb_context *tdb)
 539 {
 540         tdb->last_error = TDB_SUCCESS;
 541         tdb->tdb1.io->tdb1_oob(tdb, tdb->file->map_size, 1, true);
 542         return tdb->last_error;
 543 }