ttxml: removed cruft from tests
[ccan] / ccan / tdb2 / io.c
1  /*
2    Unix SMB/CIFS implementation.
3
4    trivial database library
5
6    Copyright (C) Andrew Tridgell              1999-2005
7    Copyright (C) Paul `Rusty' Russell              2000
8    Copyright (C) Jeremy Allison                    2000-2003
9    Copyright (C) Rusty Russell                     2010
10
11      ** NOTE! The following LGPL license applies to the tdb
12      ** library. This does NOT imply that all of Samba is released
13      ** under the LGPL
14
15    This library is free software; you can redistribute it and/or
16    modify it under the terms of the GNU Lesser General Public
17    License as published by the Free Software Foundation; either
18    version 3 of the License, or (at your option) any later version.
19
20    This library is distributed in the hope that it will be useful,
21    but WITHOUT ANY WARRANTY; without even the implied warranty of
22    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
23    Lesser General Public License for more details.
24
25    You should have received a copy of the GNU Lesser General Public
26    License along with this library; if not, see <http://www.gnu.org/licenses/>.
27 */
28 #include "private.h"
29 #include <assert.h>
30 #include <ccan/likely/likely.h>
31
32 void tdb_munmap(struct tdb_file *file)
33 {
34         if (file->fd == -1)
35                 return;
36
37         if (file->map_ptr) {
38                 munmap(file->map_ptr, file->map_size);
39                 file->map_ptr = NULL;
40         }
41 }
42
43 void tdb_mmap(struct tdb_context *tdb)
44 {
45         int mmap_flags;
46
47         if (tdb->flags & TDB_INTERNAL)
48                 return;
49
50         if (tdb->flags & TDB_NOMMAP)
51                 return;
52
53         if ((tdb->open_flags & O_ACCMODE) == O_RDONLY)
54                 mmap_flags = PROT_READ;
55         else
56                 mmap_flags = PROT_READ | PROT_WRITE;
57
58         /* size_t can be smaller than off_t. */
59         if ((size_t)tdb->file->map_size == tdb->file->map_size) {
60                 tdb->file->map_ptr = mmap(NULL, tdb->file->map_size,
61                                           mmap_flags,
62                                           MAP_SHARED, tdb->file->fd, 0);
63         } else
64                 tdb->file->map_ptr = MAP_FAILED;
65
66         /*
67          * NB. When mmap fails it returns MAP_FAILED *NOT* NULL !!!!
68          */
69         if (tdb->file->map_ptr == MAP_FAILED) {
70                 tdb->file->map_ptr = NULL;
71                 tdb_logerr(tdb, TDB_SUCCESS, TDB_LOG_WARNING,
72                            "tdb_mmap failed for size %lld (%s)",
73                            (long long)tdb->file->map_size, strerror(errno));
74         }
75 }
76
77 /* check for an out of bounds access - if it is out of bounds then
78    see if the database has been expanded by someone else and expand
79    if necessary
80    note that "len" is the minimum length needed for the db.
81
82    If probe is true, len being too large isn't a failure.
83 */
84 static enum TDB_ERROR tdb_oob(struct tdb_context *tdb,
85                               tdb_off_t off, tdb_len_t len, bool probe)
86 {
87         struct stat st;
88         enum TDB_ERROR ecode;
89
90         /* We can't hold pointers during this: we could unmap! */
91         assert(!tdb->tdb2.direct_access
92                || (tdb->flags & TDB_NOLOCK)
93                || tdb_has_expansion_lock(tdb));
94
95         if (len + off < len) {
96                 if (probe)
97                         return TDB_SUCCESS;
98
99                 return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
100                                   "tdb_oob off %llu len %llu wrap\n",
101                                   (long long)off, (long long)len);
102         }
103
104         if (len + off <= tdb->file->map_size)
105                 return TDB_SUCCESS;
106         if (tdb->flags & TDB_INTERNAL) {
107                 if (probe)
108                         return TDB_SUCCESS;
109
110                 tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
111                            "tdb_oob len %lld beyond internal"
112                            " malloc size %lld",
113                            (long long)(off + len),
114                            (long long)tdb->file->map_size);
115                 return TDB_ERR_IO;
116         }
117
118         ecode = tdb_lock_expand(tdb, F_RDLCK);
119         if (ecode != TDB_SUCCESS) {
120                 return ecode;
121         }
122
123         if (fstat(tdb->file->fd, &st) != 0) {
124                 tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
125                            "Failed to fstat file: %s", strerror(errno));
126                 tdb_unlock_expand(tdb, F_RDLCK);
127                 return TDB_ERR_IO;
128         }
129
130         tdb_unlock_expand(tdb, F_RDLCK);
131
132         if (st.st_size < off + len) {
133                 if (probe)
134                         return TDB_SUCCESS;
135
136                 tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
137                            "tdb_oob len %llu beyond eof at %zu",
138                            (long long)(off + len), st.st_size);
139                 return TDB_ERR_IO;
140         }
141
142         /* Unmap, update size, remap */
143         tdb_munmap(tdb->file);
144
145         tdb->file->map_size = st.st_size;
146         tdb_mmap(tdb);
147         return TDB_SUCCESS;
148 }
149
150 /* Endian conversion: we only ever deal with 8 byte quantities */
151 void *tdb_convert(const struct tdb_context *tdb, void *buf, tdb_len_t size)
152 {
153         assert(size % 8 == 0);
154         if (unlikely((tdb->flags & TDB_CONVERT)) && buf) {
155                 uint64_t i, *p = (uint64_t *)buf;
156                 for (i = 0; i < size / 8; i++)
157                         p[i] = bswap_64(p[i]);
158         }
159         return buf;
160 }
161
162 /* Return first non-zero offset in offset array, or end, or -ve error. */
163 /* FIXME: Return the off? */
164 uint64_t tdb_find_nonzero_off(struct tdb_context *tdb,
165                               tdb_off_t base, uint64_t start, uint64_t end)
166 {
167         uint64_t i;
168         const uint64_t *val;
169
170         /* Zero vs non-zero is the same unconverted: minor optimization. */
171         val = tdb_access_read(tdb, base + start * sizeof(tdb_off_t),
172                               (end - start) * sizeof(tdb_off_t), false);
173         if (TDB_PTR_IS_ERR(val)) {
174                 return TDB_ERR_TO_OFF(TDB_PTR_ERR(val));
175         }
176
177         for (i = 0; i < (end - start); i++) {
178                 if (val[i])
179                         break;
180         }
181         tdb_access_release(tdb, val);
182         return start + i;
183 }
184
185 /* Return first zero offset in num offset array, or num, or -ve error. */
186 uint64_t tdb_find_zero_off(struct tdb_context *tdb, tdb_off_t off,
187                            uint64_t num)
188 {
189         uint64_t i;
190         const uint64_t *val;
191
192         /* Zero vs non-zero is the same unconverted: minor optimization. */
193         val = tdb_access_read(tdb, off, num * sizeof(tdb_off_t), false);
194         if (TDB_PTR_IS_ERR(val)) {
195                 return TDB_ERR_TO_OFF(TDB_PTR_ERR(val));
196         }
197
198         for (i = 0; i < num; i++) {
199                 if (!val[i])
200                         break;
201         }
202         tdb_access_release(tdb, val);
203         return i;
204 }
205
206 enum TDB_ERROR zero_out(struct tdb_context *tdb, tdb_off_t off, tdb_len_t len)
207 {
208         char buf[8192] = { 0 };
209         void *p = tdb->tdb2.io->direct(tdb, off, len, true);
210         enum TDB_ERROR ecode = TDB_SUCCESS;
211
212         assert(!(tdb->flags & TDB_RDONLY));
213         if (TDB_PTR_IS_ERR(p)) {
214                 return TDB_PTR_ERR(p);
215         }
216         if (p) {
217                 memset(p, 0, len);
218                 return ecode;
219         }
220         while (len) {
221                 unsigned todo = len < sizeof(buf) ? len : sizeof(buf);
222                 ecode = tdb->tdb2.io->twrite(tdb, off, buf, todo);
223                 if (ecode != TDB_SUCCESS) {
224                         break;
225                 }
226                 len -= todo;
227                 off += todo;
228         }
229         return ecode;
230 }
231
232 tdb_off_t tdb_read_off(struct tdb_context *tdb, tdb_off_t off)
233 {
234         tdb_off_t ret;
235         enum TDB_ERROR ecode;
236
237         if (likely(!(tdb->flags & TDB_CONVERT))) {
238                 tdb_off_t *p = tdb->tdb2.io->direct(tdb, off, sizeof(*p),
239                                                     false);
240                 if (TDB_PTR_IS_ERR(p)) {
241                         return TDB_ERR_TO_OFF(TDB_PTR_ERR(p));
242                 }
243                 if (p)
244                         return *p;
245         }
246
247         ecode = tdb_read_convert(tdb, off, &ret, sizeof(ret));
248         if (ecode != TDB_SUCCESS) {
249                 return TDB_ERR_TO_OFF(ecode);
250         }
251         return ret;
252 }
253
254 /* write a lump of data at a specified offset */
255 static enum TDB_ERROR tdb_write(struct tdb_context *tdb, tdb_off_t off,
256                                 const void *buf, tdb_len_t len)
257 {
258         enum TDB_ERROR ecode;
259
260         if (tdb->flags & TDB_RDONLY) {
261                 return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
262                                   "Write to read-only database");
263         }
264
265         ecode = tdb->tdb2.io->oob(tdb, off, len, false);
266         if (ecode != TDB_SUCCESS) {
267                 return ecode;
268         }
269
270         if (tdb->file->map_ptr) {
271                 memcpy(off + (char *)tdb->file->map_ptr, buf, len);
272         } else {
273                 ssize_t ret;
274                 ret = pwrite(tdb->file->fd, buf, len, off);
275                 if (ret != len) {
276                         /* This shouldn't happen: we avoid sparse files. */
277                         if (ret >= 0)
278                                 errno = ENOSPC;
279
280                         return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
281                                           "tdb_write: %zi at %zu len=%zu (%s)",
282                                           ret, (size_t)off, (size_t)len,
283                                           strerror(errno));
284                 }
285         }
286         return TDB_SUCCESS;
287 }
288
289 /* read a lump of data at a specified offset */
290 static enum TDB_ERROR tdb_read(struct tdb_context *tdb, tdb_off_t off,
291                                void *buf, tdb_len_t len)
292 {
293         enum TDB_ERROR ecode;
294
295         ecode = tdb->tdb2.io->oob(tdb, off, len, false);
296         if (ecode != TDB_SUCCESS) {
297                 return ecode;
298         }
299
300         if (tdb->file->map_ptr) {
301                 memcpy(buf, off + (char *)tdb->file->map_ptr, len);
302         } else {
303                 ssize_t r = pread(tdb->file->fd, buf, len, off);
304                 if (r != len) {
305                         return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
306                                           "tdb_read failed with %zi at %zu "
307                                           "len=%zu (%s) map_size=%zu",
308                                           r, (size_t)off, (size_t)len,
309                                           strerror(errno),
310                                           (size_t)tdb->file->map_size);
311                 }
312         }
313         return TDB_SUCCESS;
314 }
315
316 enum TDB_ERROR tdb_write_convert(struct tdb_context *tdb, tdb_off_t off,
317                                  const void *rec, size_t len)
318 {
319         enum TDB_ERROR ecode;
320
321         if (unlikely((tdb->flags & TDB_CONVERT))) {
322                 void *conv = malloc(len);
323                 if (!conv) {
324                         return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
325                                           "tdb_write: no memory converting"
326                                           " %zu bytes", len);
327                 }
328                 memcpy(conv, rec, len);
329                 ecode = tdb->tdb2.io->twrite(tdb, off,
330                                            tdb_convert(tdb, conv, len), len);
331                 free(conv);
332         } else {
333                 ecode = tdb->tdb2.io->twrite(tdb, off, rec, len);
334         }
335         return ecode;
336 }
337
338 enum TDB_ERROR tdb_read_convert(struct tdb_context *tdb, tdb_off_t off,
339                                 void *rec, size_t len)
340 {
341         enum TDB_ERROR ecode = tdb->tdb2.io->tread(tdb, off, rec, len);
342         tdb_convert(tdb, rec, len);
343         return ecode;
344 }
345
346 enum TDB_ERROR tdb_write_off(struct tdb_context *tdb,
347                              tdb_off_t off, tdb_off_t val)
348 {
349         if (tdb->flags & TDB_RDONLY) {
350                 return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
351                                   "Write to read-only database");
352         }
353
354         if (likely(!(tdb->flags & TDB_CONVERT))) {
355                 tdb_off_t *p = tdb->tdb2.io->direct(tdb, off, sizeof(*p),
356                                                     true);
357                 if (TDB_PTR_IS_ERR(p)) {
358                         return TDB_PTR_ERR(p);
359                 }
360                 if (p) {
361                         *p = val;
362                         return TDB_SUCCESS;
363                 }
364         }
365         return tdb_write_convert(tdb, off, &val, sizeof(val));
366 }
367
368 static void *_tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset,
369                              tdb_len_t len, unsigned int prefix)
370 {
371         unsigned char *buf;
372         enum TDB_ERROR ecode;
373
374         /* some systems don't like zero length malloc */
375         buf = malloc(prefix + len ? prefix + len : 1);
376         if (!buf) {
377                 tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_USE_ERROR,
378                            "tdb_alloc_read malloc failed len=%zu",
379                            (size_t)(prefix + len));
380                 return TDB_ERR_PTR(TDB_ERR_OOM);
381         } else {
382                 ecode = tdb->tdb2.io->tread(tdb, offset, buf+prefix, len);
383                 if (unlikely(ecode != TDB_SUCCESS)) {
384                         free(buf);
385                         return TDB_ERR_PTR(ecode);
386                 }
387         }
388         return buf;
389 }
390
391 /* read a lump of data, allocating the space for it */
392 void *tdb_alloc_read(struct tdb_context *tdb, tdb_off_t offset, tdb_len_t len)
393 {
394         return _tdb_alloc_read(tdb, offset, len, 0);
395 }
396
397 static enum TDB_ERROR fill(struct tdb_context *tdb,
398                            const void *buf, size_t size,
399                            tdb_off_t off, tdb_len_t len)
400 {
401         while (len) {
402                 size_t n = len > size ? size : len;
403                 ssize_t ret = pwrite(tdb->file->fd, buf, n, off);
404                 if (ret != n) {
405                         if (ret >= 0)
406                                 errno = ENOSPC;
407
408                         return tdb_logerr(tdb, TDB_ERR_IO, TDB_LOG_ERROR,
409                                           "fill failed:"
410                                           " %zi at %zu len=%zu (%s)",
411                                           ret, (size_t)off, (size_t)len,
412                                           strerror(errno));
413                 }
414                 len -= n;
415                 off += n;
416         }
417         return TDB_SUCCESS;
418 }
419
420 /* expand a file.  we prefer to use ftruncate, as that is what posix
421   says to use for mmap expansion */
422 static enum TDB_ERROR tdb_expand_file(struct tdb_context *tdb,
423                                       tdb_len_t addition)
424 {
425         char buf[8192];
426         enum TDB_ERROR ecode;
427
428         if (tdb->flags & TDB_RDONLY) {
429                 return tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
430                                   "Expand on read-only database");
431         }
432
433         if (tdb->flags & TDB_INTERNAL) {
434                 char *new = realloc(tdb->file->map_ptr,
435                                     tdb->file->map_size + addition);
436                 if (!new) {
437                         return tdb_logerr(tdb, TDB_ERR_OOM, TDB_LOG_ERROR,
438                                           "No memory to expand database");
439                 }
440                 tdb->file->map_ptr = new;
441                 tdb->file->map_size += addition;
442         } else {
443                 /* Unmap before trying to write; old TDB claimed OpenBSD had
444                  * problem with this otherwise. */
445                 tdb_munmap(tdb->file);
446
447                 /* If this fails, we try to fill anyway. */
448                 if (ftruncate(tdb->file->fd, tdb->file->map_size + addition))
449                         ;
450
451                 /* now fill the file with something. This ensures that the
452                    file isn't sparse, which would be very bad if we ran out of
453                    disk. This must be done with write, not via mmap */
454                 memset(buf, 0x43, sizeof(buf));
455                 ecode = fill(tdb, buf, sizeof(buf), tdb->file->map_size,
456                              addition);
457                 if (ecode != TDB_SUCCESS)
458                         return ecode;
459                 tdb->file->map_size += addition;
460                 tdb_mmap(tdb);
461         }
462         return TDB_SUCCESS;
463 }
464
465 const void *tdb_access_read(struct tdb_context *tdb,
466                             tdb_off_t off, tdb_len_t len, bool convert)
467 {
468         void *ret = NULL;
469
470         if (likely(!(tdb->flags & TDB_CONVERT))) {
471                 ret = tdb->tdb2.io->direct(tdb, off, len, false);
472
473                 if (TDB_PTR_IS_ERR(ret)) {
474                         return ret;
475                 }
476         }
477         if (!ret) {
478                 struct tdb_access_hdr *hdr;
479                 hdr = _tdb_alloc_read(tdb, off, len, sizeof(*hdr));
480                 if (TDB_PTR_IS_ERR(hdr)) {
481                         return hdr;
482                 }
483                 hdr->next = tdb->tdb2.access;
484                 tdb->tdb2.access = hdr;
485                 ret = hdr + 1;
486                 if (convert) {
487                         tdb_convert(tdb, (void *)ret, len);
488                 }
489         } else
490                 tdb->tdb2.direct_access++;
491
492         return ret;
493 }
494
495 void *tdb_access_write(struct tdb_context *tdb,
496                        tdb_off_t off, tdb_len_t len, bool convert)
497 {
498         void *ret = NULL;
499
500         if (tdb->flags & TDB_RDONLY) {
501                 tdb_logerr(tdb, TDB_ERR_RDONLY, TDB_LOG_USE_ERROR,
502                            "Write to read-only database");
503                 return TDB_ERR_PTR(TDB_ERR_RDONLY);
504         }
505
506         if (likely(!(tdb->flags & TDB_CONVERT))) {
507                 ret = tdb->tdb2.io->direct(tdb, off, len, true);
508
509                 if (TDB_PTR_IS_ERR(ret)) {
510                         return ret;
511                 }
512         }
513
514         if (!ret) {
515                 struct tdb_access_hdr *hdr;
516                 hdr = _tdb_alloc_read(tdb, off, len, sizeof(*hdr));
517                 if (TDB_PTR_IS_ERR(hdr)) {
518                         return hdr;
519                 }
520                 hdr->next = tdb->tdb2.access;
521                 tdb->tdb2.access = hdr;
522                 hdr->off = off;
523                 hdr->len = len;
524                 hdr->convert = convert;
525                 ret = hdr + 1;
526                 if (convert)
527                         tdb_convert(tdb, (void *)ret, len);
528         } else
529                 tdb->tdb2.direct_access++;
530
531         return ret;
532 }
533
534 static struct tdb_access_hdr **find_hdr(struct tdb_context *tdb, const void *p)
535 {
536         struct tdb_access_hdr **hp;
537
538         for (hp = &tdb->tdb2.access; *hp; hp = &(*hp)->next) {
539                 if (*hp + 1 == p)
540                         return hp;
541         }
542         return NULL;
543 }
544
545 void tdb_access_release(struct tdb_context *tdb, const void *p)
546 {
547         struct tdb_access_hdr *hdr, **hp = find_hdr(tdb, p);
548
549         if (hp) {
550                 hdr = *hp;
551                 *hp = hdr->next;
552                 free(hdr);
553         } else
554                 tdb->tdb2.direct_access--;
555 }
556
557 enum TDB_ERROR tdb_access_commit(struct tdb_context *tdb, void *p)
558 {
559         struct tdb_access_hdr *hdr, **hp = find_hdr(tdb, p);
560         enum TDB_ERROR ecode;
561
562         if (hp) {
563                 hdr = *hp;
564                 if (hdr->convert)
565                         ecode = tdb_write_convert(tdb, hdr->off, p, hdr->len);
566                 else
567                         ecode = tdb_write(tdb, hdr->off, p, hdr->len);
568                 *hp = hdr->next;
569                 free(hdr);
570         } else {
571                 tdb->tdb2.direct_access--;
572                 ecode = TDB_SUCCESS;
573         }
574
575         return ecode;
576 }
577
578 static void *tdb_direct(struct tdb_context *tdb, tdb_off_t off, size_t len,
579                         bool write_mode)
580 {
581         enum TDB_ERROR ecode;
582
583         if (unlikely(!tdb->file->map_ptr))
584                 return NULL;
585
586         ecode = tdb_oob(tdb, off, len, false);
587         if (unlikely(ecode != TDB_SUCCESS))
588                 return TDB_ERR_PTR(ecode);
589         return (char *)tdb->file->map_ptr + off;
590 }
591
592 void tdb_inc_seqnum(struct tdb_context *tdb)
593 {
594         tdb_off_t seq;
595
596         if (tdb->flags & TDB_VERSION1) {
597                 tdb1_increment_seqnum_nonblock(tdb);
598                 return;
599         }
600
601         if (likely(!(tdb->flags & TDB_CONVERT))) {
602                 int64_t *direct;
603
604                 direct = tdb->tdb2.io->direct(tdb,
605                                               offsetof(struct tdb_header,
606                                                        seqnum),
607                                               sizeof(*direct), true);
608                 if (likely(direct)) {
609                         /* Don't let it go negative, even briefly */
610                         if (unlikely((*direct) + 1) < 0)
611                                 *direct = 0;
612                         (*direct)++;
613                         return;
614                 }
615         }
616
617         seq = tdb_read_off(tdb, offsetof(struct tdb_header, seqnum));
618         if (!TDB_OFF_IS_ERR(seq)) {
619                 seq++;
620                 if (unlikely((int64_t)seq < 0))
621                         seq = 0;
622                 tdb_write_off(tdb, offsetof(struct tdb_header, seqnum), seq);
623         }
624 }
625
626 static const struct tdb_methods io_methods = {
627         tdb_read,
628         tdb_write,
629         tdb_oob,
630         tdb_expand_file,
631         tdb_direct,
632 };
633
634 /*
635   initialise the default methods table
636 */
637 void tdb_io_init(struct tdb_context *tdb)
638 {
639         tdb->tdb2.io = &io_methods;
640 }