Fix missing va_end()s
[ccan] / ccan / ntdb / open.c
1  /*
2    Trivial Database 2: opening and closing TDBs
3    Copyright (C) Rusty Russell 2010
4
5    This library is free software; you can redistribute it and/or
6    modify it under the terms of the GNU Lesser General Public
7    License as published by the Free Software Foundation; either
8    version 3 of the License, or (at your option) any later version.
9
10    This library is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13    Lesser General Public License for more details.
14
15    You should have received a copy of the GNU Lesser General Public
16    License along with this library; if not, see <http://www.gnu.org/licenses/>.
17 */
18 #include "private.h"
19 #include <ccan/build_assert/build_assert.h>
20
21 /* all tdbs, to detect double-opens (fcntl file don't nest!) */
22 static struct ntdb_context *tdbs = NULL;
23
24 static struct ntdb_file *find_file(dev_t device, ino_t ino)
25 {
26         struct ntdb_context *i;
27
28         for (i = tdbs; i; i = i->next) {
29                 if (i->file->device == device && i->file->inode == ino) {
30                         i->file->refcnt++;
31                         return i->file;
32                 }
33         }
34         return NULL;
35 }
36
37 static bool read_all(int fd, void *buf, size_t len)
38 {
39         while (len) {
40                 ssize_t ret;
41                 ret = read(fd, buf, len);
42                 if (ret < 0)
43                         return false;
44                 if (ret == 0) {
45                         /* ETOOSHORT? */
46                         errno = EWOULDBLOCK;
47                         return false;
48                 }
49                 buf = (char *)buf + ret;
50                 len -= ret;
51         }
52         return true;
53 }
54
55 static uint32_t random_number(struct ntdb_context *ntdb)
56 {
57         int fd;
58         uint32_t ret = 0;
59         struct timeval now;
60
61         fd = open("/dev/urandom", O_RDONLY);
62         if (fd >= 0) {
63                 if (read_all(fd, &ret, sizeof(ret))) {
64                         close(fd);
65                         return ret;
66                 }
67                 close(fd);
68         }
69         /* FIXME: Untested!  Based on Wikipedia protocol description! */
70         fd = open("/dev/egd-pool", O_RDWR);
71         if (fd >= 0) {
72                 /* Command is 1, next byte is size we want to read. */
73                 char cmd[2] = { 1, sizeof(uint32_t) };
74                 if (write(fd, cmd, sizeof(cmd)) == sizeof(cmd)) {
75                         char reply[1 + sizeof(uint32_t)];
76                         int r = read(fd, reply, sizeof(reply));
77                         if (r > 1) {
78                                 /* Copy at least some bytes. */
79                                 memcpy(&ret, reply+1, r - 1);
80                                 if (reply[0] == sizeof(uint32_t)
81                                     && r == sizeof(reply)) {
82                                         close(fd);
83                                         return ret;
84                                 }
85                         }
86                 }
87                 close(fd);
88         }
89
90         /* Fallback: pid and time. */
91         gettimeofday(&now, NULL);
92         ret = getpid() * 100132289ULL + now.tv_sec * 1000000ULL + now.tv_usec;
93         ntdb_logerr(ntdb, NTDB_SUCCESS, NTDB_LOG_WARNING,
94                    "ntdb_open: random from getpid and time");
95         return ret;
96 }
97
98 static void ntdb_context_init(struct ntdb_context *ntdb)
99 {
100         /* Initialize the NTDB fields here */
101         ntdb_io_init(ntdb);
102         ntdb->transaction = NULL;
103         ntdb->access = NULL;
104 }
105
106 /* initialise a new database:
107  *
108  *      struct ntdb_header;
109  *      struct {
110  *              struct ntdb_used_record hash_header;
111  *              ntdb_off_t hash_buckets[1 << ntdb->hash_bits];
112  *      } hash;
113  *      struct ntdb_freetable ftable;
114  *      struct {
115  *              struct ntdb_free_record free_header;
116  *              char forty_three[...];
117  *      } remainder;
118  */
119 #define NEW_DATABASE_HDR_SIZE(hbits)                                    \
120         (sizeof(struct ntdb_header)                                     \
121          + sizeof(struct ntdb_used_record) + (sizeof(ntdb_off_t) << hbits) \
122          + sizeof(struct ntdb_freetable)                                \
123          + sizeof(struct ntdb_free_record))
124
125 static enum NTDB_ERROR ntdb_new_database(struct ntdb_context *ntdb,
126                                          struct ntdb_attribute_seed *seed,
127                                          struct ntdb_header *rhdr)
128 {
129         /* We make it up in memory, then write it out if not internal */
130         struct ntdb_freetable *ftable;
131         struct ntdb_used_record *htable;
132         struct ntdb_header *hdr;
133         struct ntdb_free_record *remainder;
134         char *mem;
135         unsigned int magic_len;
136         ssize_t rlen;
137         size_t dbsize, hashsize, hdrsize, remaindersize;
138         enum NTDB_ERROR ecode;
139
140         hashsize = sizeof(ntdb_off_t) << ntdb->hash_bits;
141
142         /* Always make db a multiple of NTDB_PGSIZE */
143         hdrsize = NEW_DATABASE_HDR_SIZE(ntdb->hash_bits);
144         dbsize = (hdrsize + NTDB_PGSIZE-1) & ~(NTDB_PGSIZE-1);
145
146         mem = ntdb->alloc_fn(ntdb, dbsize, ntdb->alloc_data);
147         if (!mem) {
148                 return ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
149                                    "ntdb_new_database: failed to allocate");
150         }
151
152         hdr = (void *)mem;
153         htable = (void *)(mem + sizeof(*hdr));
154         ftable = (void *)(mem + sizeof(*hdr) + sizeof(*htable) + hashsize);
155         remainder = (void *)(mem + sizeof(*hdr) + sizeof(*htable) + hashsize
156                              + sizeof(*ftable));
157
158         /* Fill in the header */
159         hdr->version = NTDB_VERSION;
160         if (seed)
161                 hdr->hash_seed = seed->seed;
162         else
163                 hdr->hash_seed = random_number(ntdb);
164         hdr->hash_test = NTDB_HASH_MAGIC;
165         hdr->hash_test = ntdb->hash_fn(&hdr->hash_test,
166                                        sizeof(hdr->hash_test),
167                                        hdr->hash_seed,
168                                        ntdb->hash_data);
169         hdr->hash_bits = ntdb->hash_bits;
170         hdr->recovery = 0;
171         hdr->features_used = hdr->features_offered = NTDB_FEATURE_MASK;
172         hdr->seqnum = 0;
173         hdr->capabilities = 0;
174         memset(hdr->reserved, 0, sizeof(hdr->reserved));
175
176         /* Hash is all zero after header. */
177         set_header(NULL, htable, NTDB_HTABLE_MAGIC, 0, hashsize, hashsize);
178         memset(htable + 1, 0, hashsize);
179
180         /* Free is empty. */
181         hdr->free_table = (char *)ftable - (char *)hdr;
182         memset(ftable, 0, sizeof(*ftable));
183         ecode = set_header(NULL, &ftable->hdr, NTDB_FTABLE_MAGIC, 0,
184                            sizeof(*ftable) - sizeof(ftable->hdr),
185                            sizeof(*ftable) - sizeof(ftable->hdr));
186         if (ecode != NTDB_SUCCESS) {
187                 goto out;
188         }
189
190         /* Rest of database is a free record, containing junk. */
191         remaindersize = dbsize - hdrsize;
192         remainder->ftable_and_len
193                 = (remaindersize + sizeof(*remainder)
194                    - sizeof(struct ntdb_used_record));
195         remainder->next = 0;
196         remainder->magic_and_prev
197                 = (NTDB_FREE_MAGIC << (64-NTDB_OFF_UPPER_STEAL))
198                 | ((char *)remainder - (char *)hdr);
199         memset(remainder + 1, 0x43, remaindersize);
200
201         /* Put in our single free entry. */
202         ftable->buckets[size_to_bucket(remaindersize)] =
203                 (char *)remainder - (char *)hdr;
204
205         /* Magic food */
206         memset(hdr->magic_food, 0, sizeof(hdr->magic_food));
207         strcpy(hdr->magic_food, NTDB_MAGIC_FOOD);
208
209         /* This creates an endian-converted database, as if read from disk */
210         magic_len = sizeof(hdr->magic_food);
211         ntdb_convert(ntdb, (char *)hdr + magic_len, hdrsize - magic_len);
212
213         /* Return copy of header. */
214         *rhdr = *hdr;
215
216         if (ntdb->flags & NTDB_INTERNAL) {
217                 ntdb->file->map_size = dbsize;
218                 ntdb->file->map_ptr = hdr;
219                 return NTDB_SUCCESS;
220         }
221         if (lseek(ntdb->file->fd, 0, SEEK_SET) == -1) {
222                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
223                                     "ntdb_new_database:"
224                                     " failed to seek: %s", strerror(errno));
225                 goto out;
226         }
227
228         if (ftruncate(ntdb->file->fd, 0) == -1) {
229                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
230                                     "ntdb_new_database:"
231                                     " failed to truncate: %s", strerror(errno));
232                 goto out;
233         }
234
235         rlen = write(ntdb->file->fd, hdr, dbsize);
236         if (rlen != dbsize) {
237                 if (rlen >= 0)
238                         errno = ENOSPC;
239                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
240                                     "ntdb_new_database: %zi writing header: %s",
241                                     rlen, strerror(errno));
242                 goto out;
243         }
244
245 out:
246         ntdb->free_fn(hdr, ntdb->alloc_data);
247         return ecode;
248 }
249
250 static enum NTDB_ERROR ntdb_new_file(struct ntdb_context *ntdb)
251 {
252         ntdb->file = ntdb->alloc_fn(NULL, sizeof(*ntdb->file), ntdb->alloc_data);
253         if (!ntdb->file)
254                 return ntdb_logerr(ntdb, NTDB_ERR_OOM, NTDB_LOG_ERROR,
255                                   "ntdb_open: cannot alloc ntdb_file structure");
256         ntdb->file->num_lockrecs = 0;
257         ntdb->file->lockrecs = NULL;
258         ntdb->file->allrecord_lock.count = 0;
259         ntdb->file->refcnt = 1;
260         ntdb->file->map_ptr = NULL;
261         ntdb->file->direct_count = 0;
262         ntdb->file->old_mmaps = NULL;
263         return NTDB_SUCCESS;
264 }
265
266 _PUBLIC_ enum NTDB_ERROR ntdb_set_attribute(struct ntdb_context *ntdb,
267                                  const union ntdb_attribute *attr)
268 {
269         switch (attr->base.attr) {
270         case NTDB_ATTRIBUTE_LOG:
271                 ntdb->log_fn = attr->log.fn;
272                 ntdb->log_data = attr->log.data;
273                 break;
274         case NTDB_ATTRIBUTE_HASH:
275         case NTDB_ATTRIBUTE_SEED:
276         case NTDB_ATTRIBUTE_OPENHOOK:
277         case NTDB_ATTRIBUTE_HASHSIZE:
278                 return ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
279                                    NTDB_LOG_USE_ERROR,
280                                    "ntdb_set_attribute:"
281                                    " cannot set %s after opening",
282                                    attr->base.attr == NTDB_ATTRIBUTE_HASH
283                                    ? "NTDB_ATTRIBUTE_HASH"
284                                    : attr->base.attr == NTDB_ATTRIBUTE_SEED
285                                    ? "NTDB_ATTRIBUTE_SEED"
286                                    : attr->base.attr == NTDB_ATTRIBUTE_OPENHOOK
287                                    ? "NTDB_ATTRIBUTE_OPENHOOK"
288                                    : "NTDB_ATTRIBUTE_HASHSIZE");
289         case NTDB_ATTRIBUTE_STATS:
290                 return ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
291                                    NTDB_LOG_USE_ERROR,
292                                    "ntdb_set_attribute:"
293                                    " cannot set NTDB_ATTRIBUTE_STATS");
294         case NTDB_ATTRIBUTE_FLOCK:
295                 ntdb->lock_fn = attr->flock.lock;
296                 ntdb->unlock_fn = attr->flock.unlock;
297                 ntdb->lock_data = attr->flock.data;
298                 break;
299         case NTDB_ATTRIBUTE_ALLOCATOR:
300                 ntdb->alloc_fn = attr->alloc.alloc;
301                 ntdb->expand_fn = attr->alloc.expand;
302                 ntdb->free_fn = attr->alloc.free;
303                 ntdb->alloc_data = attr->alloc.priv_data;
304                 break;
305         default:
306                 return ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
307                                    NTDB_LOG_USE_ERROR,
308                                    "ntdb_set_attribute:"
309                                    " unknown attribute type %u",
310                                    attr->base.attr);
311         }
312         return NTDB_SUCCESS;
313 }
314
315 _PUBLIC_ enum NTDB_ERROR ntdb_get_attribute(struct ntdb_context *ntdb,
316                                  union ntdb_attribute *attr)
317 {
318         switch (attr->base.attr) {
319         case NTDB_ATTRIBUTE_LOG:
320                 if (!ntdb->log_fn)
321                         return NTDB_ERR_NOEXIST;
322                 attr->log.fn = ntdb->log_fn;
323                 attr->log.data = ntdb->log_data;
324                 break;
325         case NTDB_ATTRIBUTE_HASH:
326                 attr->hash.fn = ntdb->hash_fn;
327                 attr->hash.data = ntdb->hash_data;
328                 break;
329         case NTDB_ATTRIBUTE_SEED:
330                 attr->seed.seed = ntdb->hash_seed;
331                 break;
332         case NTDB_ATTRIBUTE_OPENHOOK:
333                 if (!ntdb->openhook)
334                         return NTDB_ERR_NOEXIST;
335                 attr->openhook.fn = ntdb->openhook;
336                 attr->openhook.data = ntdb->openhook_data;
337                 break;
338         case NTDB_ATTRIBUTE_STATS: {
339                 size_t size = attr->stats.size;
340                 if (size > ntdb->stats.size)
341                         size = ntdb->stats.size;
342                 memcpy(&attr->stats, &ntdb->stats, size);
343                 break;
344         }
345         case NTDB_ATTRIBUTE_FLOCK:
346                 attr->flock.lock = ntdb->lock_fn;
347                 attr->flock.unlock = ntdb->unlock_fn;
348                 attr->flock.data = ntdb->lock_data;
349                 break;
350         case NTDB_ATTRIBUTE_ALLOCATOR:
351                 attr->alloc.alloc = ntdb->alloc_fn;
352                 attr->alloc.expand = ntdb->expand_fn;
353                 attr->alloc.free = ntdb->free_fn;
354                 attr->alloc.priv_data = ntdb->alloc_data;
355                 break;
356         case NTDB_ATTRIBUTE_HASHSIZE:
357                 attr->hashsize.size = 1 << ntdb->hash_bits;
358                 break;
359         default:
360                 return ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
361                                    NTDB_LOG_USE_ERROR,
362                                    "ntdb_get_attribute:"
363                                    " unknown attribute type %u",
364                                    attr->base.attr);
365         }
366         attr->base.next = NULL;
367         return NTDB_SUCCESS;
368 }
369
370 _PUBLIC_ void ntdb_unset_attribute(struct ntdb_context *ntdb,
371                          enum ntdb_attribute_type type)
372 {
373         switch (type) {
374         case NTDB_ATTRIBUTE_LOG:
375                 ntdb->log_fn = NULL;
376                 break;
377         case NTDB_ATTRIBUTE_OPENHOOK:
378                 ntdb->openhook = NULL;
379                 break;
380         case NTDB_ATTRIBUTE_HASH:
381         case NTDB_ATTRIBUTE_SEED:
382                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
383                            "ntdb_unset_attribute: cannot unset %s after opening",
384                            type == NTDB_ATTRIBUTE_HASH
385                            ? "NTDB_ATTRIBUTE_HASH"
386                            : "NTDB_ATTRIBUTE_SEED");
387                 break;
388         case NTDB_ATTRIBUTE_STATS:
389                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
390                            NTDB_LOG_USE_ERROR,
391                            "ntdb_unset_attribute:"
392                            "cannot unset NTDB_ATTRIBUTE_STATS");
393                 break;
394         case NTDB_ATTRIBUTE_FLOCK:
395                 ntdb->lock_fn = ntdb_fcntl_lock;
396                 ntdb->unlock_fn = ntdb_fcntl_unlock;
397                 break;
398         default:
399                 ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
400                            NTDB_LOG_USE_ERROR,
401                            "ntdb_unset_attribute: unknown attribute type %u",
402                            type);
403         }
404 }
405
406 /* The top three bits of the capability tell us whether it matters. */
407 enum NTDB_ERROR unknown_capability(struct ntdb_context *ntdb, const char *caller,
408                                   ntdb_off_t type)
409 {
410         if (type & NTDB_CAP_NOOPEN) {
411                 return ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
412                                   "%s: file has unknown capability %llu",
413                                   caller, type & NTDB_CAP_NOOPEN);
414         }
415
416         if ((type & NTDB_CAP_NOWRITE) && !(ntdb->flags & NTDB_RDONLY)) {
417                 return ntdb_logerr(ntdb, NTDB_ERR_RDONLY, NTDB_LOG_ERROR,
418                                   "%s: file has unknown capability %llu"
419                                   " (cannot write to it)",
420                                   caller, type & NTDB_CAP_NOOPEN);
421         }
422
423         if (type & NTDB_CAP_NOCHECK) {
424                 ntdb->flags |= NTDB_CANT_CHECK;
425         }
426         return NTDB_SUCCESS;
427 }
428
429 static enum NTDB_ERROR capabilities_ok(struct ntdb_context *ntdb,
430                                       ntdb_off_t capabilities)
431 {
432         ntdb_off_t off, next;
433         enum NTDB_ERROR ecode = NTDB_SUCCESS;
434         const struct ntdb_capability *cap;
435
436         /* Check capability list. */
437         for (off = capabilities; off && ecode == NTDB_SUCCESS; off = next) {
438                 cap = ntdb_access_read(ntdb, off, sizeof(*cap), true);
439                 if (NTDB_PTR_IS_ERR(cap)) {
440                         return NTDB_PTR_ERR(cap);
441                 }
442
443                 switch (cap->type & NTDB_CAP_TYPE_MASK) {
444                 /* We don't understand any capabilities (yet). */
445                 default:
446                         ecode = unknown_capability(ntdb, "ntdb_open", cap->type);
447                 }
448                 next = cap->next;
449                 ntdb_access_release(ntdb, cap);
450         }
451         return ecode;
452 }
453
454 static void *default_alloc(const void *owner, size_t len, void *priv_data)
455 {
456         return malloc(len);
457 }
458
459 static void *default_expand(void *ptr, size_t len, void *priv_data)
460 {
461         return realloc(ptr, len);
462 }
463
464 static void default_free(void *ptr, void *priv_data)
465 {
466         free(ptr);
467 }
468
469 /* First allocation needs manual search of attributes. */
470 static struct ntdb_context *alloc_ntdb(const union ntdb_attribute *attr,
471                                        const char *name)
472 {
473         size_t len = sizeof(struct ntdb_context) + strlen(name) + 1;
474
475         while (attr) {
476                 if  (attr->base.attr == NTDB_ATTRIBUTE_ALLOCATOR) {
477                         return attr->alloc.alloc(NULL, len,
478                                                  attr->alloc.priv_data);
479                 }
480                 attr = attr->base.next;
481         }
482         return default_alloc(NULL, len, NULL);
483 }
484
485 static unsigned int next_pow2(uint64_t size)
486 {
487         unsigned int bits = 1;
488
489         while ((1ULL << bits) < size)
490                 bits++;
491         return bits;
492 }
493
494 _PUBLIC_ struct ntdb_context *ntdb_open(const char *name, int ntdb_flags,
495                                         int open_flags, mode_t mode,
496                                         union ntdb_attribute *attr)
497 {
498         struct ntdb_context *ntdb;
499         struct stat st;
500         int saved_errno = 0;
501         uint64_t hash_test;
502         unsigned v;
503         ssize_t rlen;
504         struct ntdb_header hdr;
505         struct ntdb_attribute_seed *seed = NULL;
506         ntdb_bool_err berr;
507         enum NTDB_ERROR ecode;
508         int openlock;
509
510         ntdb = alloc_ntdb(attr, name);
511         if (!ntdb) {
512                 /* Can't log this */
513                 errno = ENOMEM;
514                 return NULL;
515         }
516         /* Set name immediately for logging functions. */
517         ntdb->name = strcpy((char *)(ntdb + 1), name);
518         ntdb->flags = ntdb_flags;
519         ntdb->log_fn = NULL;
520         ntdb->open_flags = open_flags;
521         ntdb->file = NULL;
522         ntdb->openhook = NULL;
523         ntdb->lock_fn = ntdb_fcntl_lock;
524         ntdb->unlock_fn = ntdb_fcntl_unlock;
525         ntdb->hash_fn = ntdb_jenkins_hash;
526         memset(&ntdb->stats, 0, sizeof(ntdb->stats));
527         ntdb->stats.base.attr = NTDB_ATTRIBUTE_STATS;
528         ntdb->stats.size = sizeof(ntdb->stats);
529         ntdb->alloc_fn = default_alloc;
530         ntdb->expand_fn = default_expand;
531         ntdb->free_fn = default_free;
532         ntdb->hash_bits = NTDB_DEFAULT_HBITS; /* 64k of hash by default. */
533
534         while (attr) {
535                 switch (attr->base.attr) {
536                 case NTDB_ATTRIBUTE_HASH:
537                         ntdb->hash_fn = attr->hash.fn;
538                         ntdb->hash_data = attr->hash.data;
539                         break;
540                 case NTDB_ATTRIBUTE_SEED:
541                         seed = &attr->seed;
542                         break;
543                 case NTDB_ATTRIBUTE_OPENHOOK:
544                         ntdb->openhook = attr->openhook.fn;
545                         ntdb->openhook_data = attr->openhook.data;
546                         break;
547                 case NTDB_ATTRIBUTE_HASHSIZE:
548                         ntdb->hash_bits = next_pow2(attr->hashsize.size);
549                         if (ntdb->hash_bits > 31) {
550                                 ecode = ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
551                                                     NTDB_LOG_USE_ERROR,
552                                                     "ntdb_open: hash_size %u"
553                                                     " too large",
554                                                     attr->hashsize.size);
555                                 goto fail;
556                         }
557                         break;
558                 default:
559                         /* These are set as normal. */
560                         ecode = ntdb_set_attribute(ntdb, attr);
561                         if (ecode != NTDB_SUCCESS)
562                                 goto fail;
563                 }
564                 attr = attr->base.next;
565         }
566
567         if (ntdb_flags & ~(NTDB_INTERNAL | NTDB_NOLOCK | NTDB_NOMMAP | NTDB_CONVERT
568                           | NTDB_NOSYNC | NTDB_SEQNUM | NTDB_ALLOW_NESTING
569                           | NTDB_RDONLY)) {
570                 ecode = ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
571                                    "ntdb_open: unknown flags %u", ntdb_flags);
572                 goto fail;
573         }
574
575         if (seed) {
576                 if (!(ntdb_flags & NTDB_INTERNAL) && !(open_flags & O_CREAT)) {
577                         ecode = ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
578                                            NTDB_LOG_USE_ERROR,
579                                            "ntdb_open:"
580                                            " cannot set NTDB_ATTRIBUTE_SEED"
581                                            " without O_CREAT.");
582                         goto fail;
583                 }
584         }
585
586         if ((open_flags & O_ACCMODE) == O_WRONLY) {
587                 ecode = ntdb_logerr(ntdb, NTDB_ERR_EINVAL, NTDB_LOG_USE_ERROR,
588                                    "ntdb_open: can't open ntdb %s write-only",
589                                    name);
590                 goto fail;
591         }
592
593         if ((open_flags & O_ACCMODE) == O_RDONLY) {
594                 openlock = F_RDLCK;
595                 ntdb->flags |= NTDB_RDONLY;
596         } else {
597                 if (ntdb_flags & NTDB_RDONLY) {
598                         ecode = ntdb_logerr(ntdb, NTDB_ERR_EINVAL,
599                                            NTDB_LOG_USE_ERROR,
600                                            "ntdb_open: can't use NTDB_RDONLY"
601                                            " without O_RDONLY");
602                         goto fail;
603                 }
604                 openlock = F_WRLCK;
605         }
606
607         /* internal databases don't need any of the rest. */
608         if (ntdb->flags & NTDB_INTERNAL) {
609                 ntdb->flags |= (NTDB_NOLOCK | NTDB_NOMMAP);
610                 ecode = ntdb_new_file(ntdb);
611                 if (ecode != NTDB_SUCCESS) {
612                         goto fail;
613                 }
614                 ntdb->file->fd = -1;
615                 ecode = ntdb_new_database(ntdb, seed, &hdr);
616                 if (ecode == NTDB_SUCCESS) {
617                         ntdb_convert(ntdb, &hdr.hash_seed,
618                                     sizeof(hdr.hash_seed));
619                         ntdb->hash_seed = hdr.hash_seed;
620                         ntdb_context_init(ntdb);
621                         ntdb_ftable_init(ntdb);
622                 }
623                 if (ecode != NTDB_SUCCESS) {
624                         goto fail;
625                 }
626                 return ntdb;
627         }
628
629         if (stat(name, &st) != -1)
630                 ntdb->file = find_file(st.st_dev, st.st_ino);
631
632         if (!ntdb->file) {
633                 ecode = ntdb_new_file(ntdb);
634                 if (ecode != NTDB_SUCCESS) {
635                         goto fail;
636                 }
637
638                 /* Set this now, as ntdb_nest_lock examines it. */
639                 ntdb->file->map_size = 0;
640
641                 if ((ntdb->file->fd = open(name, open_flags, mode)) == -1) {
642                         enum ntdb_log_level lvl;
643                         /* errno set by open(2) */
644                         saved_errno = errno;
645
646                         /* Probing for files like this is a common pattern. */
647                         if (!(open_flags & O_CREAT) && errno == ENOENT) {
648                                 lvl = NTDB_LOG_WARNING;
649                         } else {
650                                 lvl = NTDB_LOG_ERROR;
651                         }
652                         ntdb_logerr(ntdb, NTDB_ERR_IO, lvl,
653                                    "ntdb_open: could not open file %s: %s",
654                                    name, strerror(errno));
655
656                         goto fail_errno;
657                 }
658
659                 /* ensure there is only one process initialising at once:
660                  * do it immediately to reduce the create/openlock race. */
661                 ecode = ntdb_lock_open(ntdb, openlock,
662                                        NTDB_LOCK_WAIT|NTDB_LOCK_NOCHECK);
663                 if (ecode != NTDB_SUCCESS) {
664                         saved_errno = errno;
665                         goto fail_errno;
666                 }
667
668                 /* on exec, don't inherit the fd */
669                 v = fcntl(ntdb->file->fd, F_GETFD, 0);
670                 fcntl(ntdb->file->fd, F_SETFD, v | FD_CLOEXEC);
671
672                 if (fstat(ntdb->file->fd, &st) == -1) {
673                         saved_errno = errno;
674                         ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
675                                    "ntdb_open: could not stat open %s: %s",
676                                    name, strerror(errno));
677                         goto fail_errno;
678                 }
679
680                 ntdb->file->device = st.st_dev;
681                 ntdb->file->inode = st.st_ino;
682
683                 /* call their open hook if they gave us one. */
684                 if (ntdb->openhook) {
685                         ecode = ntdb->openhook(ntdb->file->fd, ntdb->openhook_data);
686                         if (ecode != NTDB_SUCCESS) {
687                                 ntdb_logerr(ntdb, ecode, NTDB_LOG_ERROR,
688                                             "ntdb_open: open hook failed");
689                                 goto fail;
690                         }
691                         open_flags |= O_CREAT;
692                 }
693         } else {
694                 /* ensure there is only one process initialising at once */
695                 ecode = ntdb_lock_open(ntdb, openlock,
696                                        NTDB_LOCK_WAIT|NTDB_LOCK_NOCHECK);
697                 if (ecode != NTDB_SUCCESS) {
698                         saved_errno = errno;
699                         goto fail_errno;
700                 }
701         }
702
703         /* If they used O_TRUNC, read will return 0. */
704         rlen = pread(ntdb->file->fd, &hdr, sizeof(hdr), 0);
705         if (rlen == 0 && (open_flags & O_CREAT)) {
706                 ecode = ntdb_new_database(ntdb, seed, &hdr);
707                 if (ecode != NTDB_SUCCESS) {
708                         goto fail;
709                 }
710         } else if (rlen < 0) {
711                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
712                                    "ntdb_open: error %s reading %s",
713                                    strerror(errno), name);
714                 goto fail;
715         } else if (rlen < sizeof(hdr)
716                    || strcmp(hdr.magic_food, NTDB_MAGIC_FOOD) != 0) {
717                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
718                                    "ntdb_open: %s is not a ntdb file", name);
719                 goto fail;
720         }
721
722         if (hdr.version != NTDB_VERSION) {
723                 if (hdr.version == bswap_64(NTDB_VERSION))
724                         ntdb->flags |= NTDB_CONVERT;
725                 else {
726                         /* wrong version */
727                         ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
728                                            "ntdb_open:"
729                                            " %s is unknown version 0x%llx",
730                                            name, (long long)hdr.version);
731                         goto fail;
732                 }
733         } else if (ntdb->flags & NTDB_CONVERT) {
734                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
735                                    "ntdb_open:"
736                                    " %s does not need NTDB_CONVERT",
737                                    name);
738                 goto fail;
739         }
740
741         ntdb_context_init(ntdb);
742
743         ntdb_convert(ntdb, &hdr, sizeof(hdr));
744         ntdb->hash_bits = hdr.hash_bits;
745         ntdb->hash_seed = hdr.hash_seed;
746         hash_test = NTDB_HASH_MAGIC;
747         hash_test = ntdb_hash(ntdb, &hash_test, sizeof(hash_test));
748         if (hdr.hash_test != hash_test) {
749                 /* wrong hash variant */
750                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
751                                    "ntdb_open:"
752                                    " %s uses a different hash function",
753                                    name);
754                 goto fail;
755         }
756
757         ecode = capabilities_ok(ntdb, hdr.capabilities);
758         if (ecode != NTDB_SUCCESS) {
759                 goto fail;
760         }
761
762         /* Clear any features we don't understand. */
763         if ((open_flags & O_ACCMODE) != O_RDONLY) {
764                 hdr.features_used &= NTDB_FEATURE_MASK;
765                 ecode = ntdb_write_convert(ntdb, offsetof(struct ntdb_header,
766                                                         features_used),
767                                           &hdr.features_used,
768                                           sizeof(hdr.features_used));
769                 if (ecode != NTDB_SUCCESS)
770                         goto fail;
771         }
772
773         ntdb_unlock_open(ntdb, openlock);
774
775         /* This makes sure we have current map_size and mmap. */
776         ecode = ntdb_oob(ntdb, ntdb->file->map_size, 1, true);
777         if (unlikely(ecode != NTDB_SUCCESS))
778                 goto fail;
779
780         if (ntdb->file->map_size % NTDB_PGSIZE != 0) {
781                 ecode = ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
782                                     "ntdb_open:"
783                                     " %s size %llu isn't a multiple of %u",
784                                     name, (long long)ntdb->file->map_size,
785                                     NTDB_PGSIZE);
786                 goto fail;
787         }
788
789         /* Now it's fully formed, recover if necessary. */
790         berr = ntdb_needs_recovery(ntdb);
791         if (unlikely(berr != false)) {
792                 if (berr < 0) {
793                         ecode = NTDB_OFF_TO_ERR(berr);
794                         goto fail;
795                 }
796                 ecode = ntdb_lock_and_recover(ntdb);
797                 if (ecode != NTDB_SUCCESS) {
798                         goto fail;
799                 }
800         }
801
802         ecode = ntdb_ftable_init(ntdb);
803         if (ecode != NTDB_SUCCESS) {
804                 goto fail;
805         }
806
807         ntdb->next = tdbs;
808         tdbs = ntdb;
809         return ntdb;
810
811  fail:
812         /* Map ecode to some logical errno. */
813         switch (NTDB_ERR_TO_OFF(ecode)) {
814         case NTDB_ERR_TO_OFF(NTDB_ERR_CORRUPT):
815         case NTDB_ERR_TO_OFF(NTDB_ERR_IO):
816                 saved_errno = EIO;
817                 break;
818         case NTDB_ERR_TO_OFF(NTDB_ERR_LOCK):
819                 saved_errno = EWOULDBLOCK;
820                 break;
821         case NTDB_ERR_TO_OFF(NTDB_ERR_OOM):
822                 saved_errno = ENOMEM;
823                 break;
824         case NTDB_ERR_TO_OFF(NTDB_ERR_EINVAL):
825                 saved_errno = EINVAL;
826                 break;
827         default:
828                 saved_errno = EINVAL;
829                 break;
830         }
831
832 fail_errno:
833 #ifdef NTDB_TRACE
834         close(ntdb->tracefd);
835 #endif
836         if (ntdb->file) {
837                 ntdb_lock_cleanup(ntdb);
838                 if (--ntdb->file->refcnt == 0) {
839                         assert(ntdb->file->num_lockrecs == 0);
840                         if (ntdb->file->map_ptr) {
841                                 if (ntdb->flags & NTDB_INTERNAL) {
842                                         ntdb->free_fn(ntdb->file->map_ptr,
843                                                       ntdb->alloc_data);
844                                 } else
845                                         ntdb_munmap(ntdb);
846                         }
847                         if (ntdb->file->fd != -1 && close(ntdb->file->fd) != 0)
848                                 ntdb_logerr(ntdb, NTDB_ERR_IO, NTDB_LOG_ERROR,
849                                            "ntdb_open: failed to close ntdb fd"
850                                            " on error: %s", strerror(errno));
851                         ntdb->free_fn(ntdb->file->lockrecs, ntdb->alloc_data);
852                         ntdb->free_fn(ntdb->file, ntdb->alloc_data);
853                 }
854         }
855
856         ntdb->free_fn(ntdb, ntdb->alloc_data);
857         errno = saved_errno;
858         return NULL;
859 }
860
861 _PUBLIC_ int ntdb_close(struct ntdb_context *ntdb)
862 {
863         int ret = 0;
864         struct ntdb_context **i;
865
866         ntdb_trace(ntdb, "ntdb_close");
867
868         if (ntdb->transaction) {
869                 ntdb_transaction_cancel(ntdb);
870         }
871
872         ntdb_lock_cleanup(ntdb);
873         if (--ntdb->file->refcnt == 0) {
874                 if (ntdb->file->map_ptr) {
875                         if (ntdb->flags & NTDB_INTERNAL) {
876                                 ntdb->free_fn(ntdb->file->map_ptr,
877                                               ntdb->alloc_data);
878                         } else {
879                                 ntdb_munmap(ntdb);
880                         }
881                 }
882                 ret = close(ntdb->file->fd);
883                 ntdb->free_fn(ntdb->file->lockrecs, ntdb->alloc_data);
884                 ntdb->free_fn(ntdb->file, ntdb->alloc_data);
885         }
886
887         /* Remove from tdbs list */
888         for (i = &tdbs; *i; i = &(*i)->next) {
889                 if (*i == ntdb) {
890                         *i = ntdb->next;
891                         break;
892                 }
893         }
894
895 #ifdef NTDB_TRACE
896         close(ntdb->tracefd);
897 #endif
898         ntdb->free_fn(ntdb, ntdb->alloc_data);
899
900         return ret;
901 }
902
903 _PUBLIC_ void ntdb_foreach_(int (*fn)(struct ntdb_context *, void *), void *p)
904 {
905         struct ntdb_context *i;
906
907         for (i = tdbs; i; i = i->next) {
908                 if (fn(i, p) != 0)
909                         break;
910         }
911 }