git.ozlabs.org Git - ccan/blob - ccan/failtest/failtest.c

   1 /* Licensed under LGPL - see LICENSE file for details */
   2 #include <ccan/failtest/failtest.h>
   3 #include <stdarg.h>
   4 #include <string.h>
   5 #include <stdio.h>
   6 #include <stdarg.h>
   7 #include <ctype.h>
   8 #include <err.h>
   9 #include <unistd.h>
  10 #include <poll.h>
  11 #include <errno.h>
  12 #include <sys/types.h>
  13 #include <sys/wait.h>
  14 #include <sys/stat.h>
  15 #include <sys/time.h>
  16 #include <signal.h>
  17 #include <assert.h>
  18 #include <ccan/time/time.h>
  19 #include <ccan/read_write_all/read_write_all.h>
  20 #include <ccan/failtest/failtest_proto.h>
  21 #include <ccan/build_assert/build_assert.h>
  22 #include <ccan/str/str.h>
  23
  24 enum failtest_result (*failtest_hook)(struct tlist_calls *);
  25
  26 static int tracefd = -1;
  27 static int warnfd;
  28
  29 unsigned int failtest_timeout_ms = 20000;
  30
  31 const char *failpath;
  32 const char *debugpath;
  33
  34 enum info_type {
  35         WRITE,
  36         RELEASE_LOCKS,
  37         FAILURE,
  38         SUCCESS,
  39         UNEXPECTED
  40 };
  41
  42 struct lock_info {
  43         int fd;
  44         /* end is inclusive: you can't have a 0-byte lock. */
  45         off_t start, end;
  46         int type;
  47 };
  48
  49 bool (*failtest_exit_check)(struct tlist_calls *history);
  50
  51 static struct tlist_calls history = TLIST_INIT(history);
  52 static int control_fd = -1;
  53 static struct timeval start;
  54 static unsigned int probe_count = 0;
  55
  56 static struct write_call *child_writes = NULL;
  57 static unsigned int child_writes_num = 0;
  58
  59 static pid_t lock_owner;
  60 static struct lock_info *locks = NULL;
  61 static unsigned int lock_num = 0;
  62
  63 static pid_t orig_pid;
  64
  65 static const char info_to_arg[] = "mceoxprwf";
  66
  67 /* Dummy call used for failtest_undo wrappers. */
  68 static struct failtest_call unrecorded_call;
  69
  70 static struct failtest_call *add_history_(enum failtest_call_type type,
  71                                           const char *file,
  72                                           unsigned int line,
  73                                           const void *elem,
  74                                           size_t elem_size)
  75 {
  76         struct failtest_call *call;
  77
  78         /* NULL file is how we suppress failure. */
  79         if (!file)
  80                 return &unrecorded_call;
  81
  82         call = malloc(sizeof *call);
  83         call->type = type;
  84         call->file = file;
  85         call->line = line;
  86         call->cleanup = NULL;
  87         memcpy(&call->u, elem, elem_size);
  88         tlist_add_tail(&history, call, list);
  89         return call;
  90 }
  91
  92 #define add_history(type, file, line, elem) \
  93         add_history_((type), (file), (line), (elem), sizeof(*(elem)))
  94
  95 /* We do a fake call inside a sizeof(), to check types. */
  96 #define set_cleanup(call, clean, type)                  \
  97         (call)->cleanup = (void *)((void)sizeof(clean((type *)NULL),1), (clean))
  98
  99
 100 /* Dup the fd to a high value (out of the way I hope!), and close the old fd. */
 101 static int move_fd_to_high(int fd)
 102 {
 103         int i;
 104
 105         for (i = FD_SETSIZE - 1; i >= 0; i--) {
 106                 if (fcntl(i, F_GETFL) == -1 && errno == EBADF) {
 107                         if (dup2(fd, i) == -1)
 108                                 err(1, "Failed to dup fd %i to %i", fd, i);
 109                         close(fd);
 110                         return i;
 111                 }
 112         }
 113         /* Nothing?  Really?  Er... ok? */
 114         return fd;
 115 }
 116
 117 static bool read_write_info(int fd)
 118 {
 119         struct write_call *w;
 120         char *buf;
 121
 122         /* We don't need all of this, but it's simple. */
 123         child_writes = realloc(child_writes,
 124                                (child_writes_num+1) * sizeof(child_writes[0]));
 125         w = &child_writes[child_writes_num];
 126         if (!read_all(fd, w, sizeof(*w)))
 127                 return false;
 128
 129         w->buf = buf = malloc(w->count);
 130         if (!read_all(fd, buf, w->count))
 131                 return false;
 132
 133         child_writes_num++;
 134         return true;
 135 }
 136
 137 static char *failpath_string(void)
 138 {
 139         struct failtest_call *i;
 140         char *ret = strdup("");
 141         unsigned len = 0;
 142
 143         /* Inefficient, but who cares? */
 144         tlist_for_each(&history, i, list) {
 145                 ret = realloc(ret, len + 2);
 146                 ret[len] = info_to_arg[i->type];
 147                 if (i->fail)
 148                         ret[len] = toupper(ret[len]);
 149                 ret[++len] = '\0';
 150         }
 151         return ret;
 152 }
 153
 154 static void warn_via_fd(int e, const char *fmt, va_list ap)
 155 {
 156         char *p = failpath_string();
 157
 158         vdprintf(warnfd, fmt, ap);
 159         if (e != -1)
 160                 dprintf(warnfd, ": %s", strerror(e));
 161         dprintf(warnfd, " [%s]\n", p);
 162         free(p);
 163 }
 164
 165 static void fwarn(const char *fmt, ...)
 166 {
 167         va_list ap;
 168         int e = errno;
 169
 170         va_start(ap, fmt);
 171         warn_via_fd(e, fmt, ap);
 172         va_end(ap);
 173 }
 174
 175
 176 static void fwarnx(const char *fmt, ...)
 177 {
 178         va_list ap;
 179
 180         va_start(ap, fmt);
 181         warn_via_fd(-1, fmt, ap);
 182         va_end(ap);
 183 }
 184
 185 static void tell_parent(enum info_type type)
 186 {
 187         if (control_fd != -1)
 188                 write_all(control_fd, &type, sizeof(type));
 189 }
 190
 191 static void child_fail(const char *out, size_t outlen, const char *fmt, ...)
 192 {
 193         va_list ap;
 194         char *path = failpath_string();
 195
 196         va_start(ap, fmt);
 197         vfprintf(stderr, fmt, ap);
 198         va_end(ap);
 199
 200         fprintf(stderr, "%.*s", (int)outlen, out);
 201         printf("To reproduce: --failpath=%s\n", path);
 202         free(path);
 203         tell_parent(FAILURE);
 204         exit(1);
 205 }
 206
 207 static void trace(const char *fmt, ...)
 208 {
 209         va_list ap;
 210
 211         if (tracefd == -1)
 212                 return;
 213
 214         va_start(ap, fmt);
 215         vdprintf(tracefd, fmt, ap);
 216         va_end(ap);
 217 }
 218
 219 static pid_t child;
 220
 221 static void hand_down(int signum)
 222 {
 223         kill(child, signum);
 224 }
 225
 226 static void release_locks(void)
 227 {
 228         /* Locks were never acquired/reacquired? */
 229         if (lock_owner == 0)
 230                 return;
 231
 232         /* We own them?  Release them all. */
 233         if (lock_owner == getpid()) {
 234                 unsigned int i;
 235                 struct flock fl;
 236                 fl.l_type = F_UNLCK;
 237                 fl.l_whence = SEEK_SET;
 238                 fl.l_start = 0;
 239                 fl.l_len = 0;
 240
 241                 for (i = 0; i < lock_num; i++)
 242                         fcntl(locks[i].fd, F_SETLK, &fl);
 243         } else {
 244                 /* Our parent must have them; pass request up. */
 245                 enum info_type type = RELEASE_LOCKS;
 246                 assert(control_fd != -1);
 247                 write_all(control_fd, &type, sizeof(type));
 248         }
 249         lock_owner = 0;
 250 }
 251
 252 /* off_t is a signed type.  Getting its max is non-trivial. */
 253 static off_t off_max(void)
 254 {
 255         BUILD_ASSERT(sizeof(off_t) == 4 || sizeof(off_t) == 8);
 256         if (sizeof(off_t) == 4)
 257                 return (off_t)0x7FFFFFF;
 258         else
 259                 return (off_t)0x7FFFFFFFFFFFFFFULL;
 260 }
 261
 262 static void get_locks(void)
 263 {
 264         unsigned int i;
 265         struct flock fl;
 266
 267         if (lock_owner == getpid())
 268                 return;
 269
 270         if (lock_owner != 0) {
 271                 enum info_type type = RELEASE_LOCKS;
 272                 assert(control_fd != -1);
 273                 write_all(control_fd, &type, sizeof(type));
 274         }
 275
 276         fl.l_whence = SEEK_SET;
 277
 278         for (i = 0; i < lock_num; i++) {
 279                 fl.l_type = locks[i].type;
 280                 fl.l_start = locks[i].start;
 281                 if (locks[i].end == off_max())
 282                         fl.l_len = 0;
 283                 else
 284                         fl.l_len = locks[i].end - locks[i].start + 1;
 285
 286                 if (fcntl(locks[i].fd, F_SETLKW, &fl) != 0)
 287                         abort();
 288         }
 289         lock_owner = getpid();
 290 }
 291
 292 struct saved_file {
 293         struct saved_file *next;
 294         int fd;
 295         void *contents;
 296         off_t off, len;
 297 };
 298
 299 static struct saved_file *save_file(struct saved_file *next, int fd)
 300 {
 301         struct saved_file *s = malloc(sizeof(*s));
 302
 303         s->next = next;
 304         s->fd = fd;
 305         s->off = lseek(fd, 0, SEEK_CUR);
 306         /* Special file?  Erk... */
 307         assert(s->off != -1);
 308         s->len = lseek(fd, 0, SEEK_END);
 309         lseek(fd, 0, SEEK_SET);
 310         s->contents = malloc(s->len);
 311         if (read(fd, s->contents, s->len) != s->len)
 312                 err(1, "Failed to save %zu bytes", (size_t)s->len);
 313         lseek(fd, s->off, SEEK_SET);
 314         return s;
 315 }
 316
 317 /* We have little choice but to save and restore open files: mmap means we
 318  * can really intercept changes in the child.
 319  *
 320  * We could do non-mmap'ed files on demand, however. */
 321 static struct saved_file *save_files(void)
 322 {
 323         struct saved_file *files = NULL;
 324         struct failtest_call *i;
 325
 326         /* Figure out the set of live fds. */
 327         tlist_for_each_rev(&history, i, list) {
 328                 if (i->type == FAILTEST_OPEN) {
 329                         int fd = i->u.open.ret;
 330                         /* Only do successful, writable fds. */
 331                         if (fd < 0)
 332                                 continue;
 333
 334                         /* If it was closed, cleanup == NULL. */
 335                         if (!i->cleanup)
 336                                 continue;
 337
 338                         if ((i->u.open.flags & O_RDWR) == O_RDWR) {
 339                                 files = save_file(files, fd);
 340                         } else if ((i->u.open.flags & O_WRONLY)
 341                                    == O_WRONLY) {
 342                                 /* FIXME: Handle O_WRONLY.  Open with O_RDWR? */
 343                                 abort();
 344                         }
 345                 }
 346         }
 347
 348         return files;
 349 }
 350
 351 static void restore_files(struct saved_file *s)
 352 {
 353         while (s) {
 354                 struct saved_file *next = s->next;
 355
 356                 lseek(s->fd, 0, SEEK_SET);
 357                 if (write(s->fd, s->contents, s->len) != s->len)
 358                         err(1, "Failed to restore %zu bytes", (size_t)s->len);
 359                 if (ftruncate(s->fd, s->len) != 0)
 360                         err(1, "Failed to trim file to length %zu",
 361                             (size_t)s->len);
 362                 free(s->contents);
 363                 lseek(s->fd, s->off, SEEK_SET);
 364                 free(s);
 365                 s = next;
 366         }
 367 }
 368
 369 static void free_files(struct saved_file *s)
 370 {
 371         while (s) {
 372                 struct saved_file *next = s->next;
 373                 free(s->contents);
 374                 free(s);
 375                 s = next;
 376         }
 377 }
 378
 379 static void free_call(struct failtest_call *call)
 380 {
 381         /* We don't do this in cleanup: needed even for failed opens. */
 382         if (call->type == FAILTEST_OPEN)
 383                 free((char *)call->u.open.pathname);
 384         tlist_del_from(&history, call, list);
 385         free(call);
 386 }
 387
 388 /* Free up memory, so valgrind doesn't report leaks. */
 389 static void free_everything(void)
 390 {
 391         struct failtest_call *i;
 392
 393         while ((i = tlist_top(&history, struct failtest_call, list)) != NULL)
 394                 free_call(i);
 395 }
 396
 397 static NORETURN void failtest_cleanup(bool forced_cleanup, int status)
 398 {
 399         struct failtest_call *i;
 400
 401         /* For children, we don't care if they "failed" the testing. */
 402         if (control_fd != -1)
 403                 status = 0;
 404
 405         if (forced_cleanup) {
 406                 /* We didn't actually do final operation: remove it. */
 407                 i = tlist_tail(&history, struct failtest_call, list);
 408                 free_call(i);
 409         }
 410
 411         /* Cleanup everything, in reverse order. */
 412         tlist_for_each_rev(&history, i, list) {
 413                 if (!i->cleanup)
 414                         continue;
 415                 if (!forced_cleanup) {
 416                         printf("Leak at %s:%u: --failpath=%s\n",
 417                                i->file, i->line, failpath_string());
 418                         status = 1;
 419                 }
 420                 i->cleanup(&i->u);
 421         }
 422
 423         free_everything();
 424         tell_parent(SUCCESS);
 425         exit(status);
 426 }
 427
 428 static bool should_fail(struct failtest_call *call)
 429 {
 430         int status;
 431         int control[2], output[2];
 432         enum info_type type = UNEXPECTED;
 433         char *out = NULL;
 434         size_t outlen = 0;
 435         struct saved_file *files;
 436
 437         /* Are we probing? */
 438         if (probe_count && --probe_count == 0 && control_fd != -1)
 439                 failtest_cleanup(true, 0);
 440
 441         if (call == &unrecorded_call)
 442                 return false;
 443
 444         if (failpath) {
 445                 /* + means continue after end, like normal. */
 446                 if (*failpath == '+')
 447                         failpath = NULL;
 448                 else if (*failpath == '\0') {
 449                         /* Continue, but don't inject errors. */
 450                         return call->fail = false;
 451                 } else {
 452                         if (tolower((unsigned char)*failpath)
 453                             != info_to_arg[call->type])
 454                                 errx(1, "Failpath expected '%c' got '%c'\n",
 455                                      info_to_arg[call->type], *failpath);
 456                         call->fail = cisupper(*(failpath++));
 457                         return call->fail;
 458                 }
 459         }
 460
 461         /* Attach debugger if they asked for it. */
 462         if (debugpath) {
 463                 char *path = failpath_string();
 464
 465                 if (streq(path, debugpath)) {
 466                         char str[80];
 467
 468                         /* Don't timeout. */
 469                         signal(SIGUSR1, SIG_IGN);
 470                         sprintf(str, "xterm -e gdb /proc/%d/exe %d &",
 471                                 getpid(), getpid());
 472                         if (system(str) == 0)
 473                                 sleep(5);
 474                 } else if (!strstarts(path, debugpath)) {
 475                         fprintf(stderr, "--debugpath not followed: %s\n", path);
 476                         debugpath = NULL;
 477                 }
 478                 free(path);
 479         }
 480
 481         if (failtest_hook) {
 482                 switch (failtest_hook(&history)) {
 483                 case FAIL_OK:
 484                         break;
 485                 case FAIL_PROBE:
 486                         /* Already down probe path?  Stop now. */
 487                         if (!probe_count) {
 488                                 /* FIXME: We should run *parent* and
 489                                  * run probe until calls match up again. */
 490                                 probe_count = 3;
 491                                 break;
 492                         } else {
 493                                 /* Child should give up now. */
 494                                 if (control_fd != -1)
 495                                         failtest_cleanup(true, 0);
 496                                 /* Parent, don't fail again. */
 497                         }
 498                 case FAIL_DONT_FAIL:
 499                         call->fail = false;
 500                         return false;
 501                 default:
 502                         abort();
 503                 }
 504         }
 505
 506         files = save_files();
 507
 508         /* We're going to fail in the child. */
 509         call->fail = true;
 510         if (pipe(control) != 0 || pipe(output) != 0)
 511                 err(1, "opening pipe");
 512
 513         /* Prevent double-printing (in child and parent) */
 514         fflush(stdout);
 515         child = fork();
 516         if (child == -1)
 517                 err(1, "forking failed");
 518
 519         if (child == 0) {
 520                 if (tracefd != -1) {
 521                         struct timeval diff;
 522                         const char *p;
 523                         char *failpath;
 524                         struct failtest_call *c;
 525
 526                         c = tlist_tail(&history, struct failtest_call, list);
 527                         diff = time_sub(time_now(), start);
 528                         failpath = failpath_string();
 529                         trace("%u->%u (%u.%02u): %s (", getppid(), getpid(),
 530                               (int)diff.tv_sec, (int)diff.tv_usec / 10000,
 531                               failpath);
 532                         free(failpath);
 533                         p = strrchr(c->file, '/');
 534                         if (p)
 535                                 trace("%s", p+1);
 536                         else
 537                                 trace("%s", c->file);
 538                         trace(":%u)\n", c->line);
 539                 }
 540                 close(control[0]);
 541                 close(output[0]);
 542                 dup2(output[1], STDOUT_FILENO);
 543                 dup2(output[1], STDERR_FILENO);
 544                 if (output[1] != STDOUT_FILENO && output[1] != STDERR_FILENO)
 545                         close(output[1]);
 546                 control_fd = move_fd_to_high(control[1]);
 547                 /* Valgrind spots the leak if we don't free these. */
 548                 free_files(files);
 549                 return true;
 550         }
 551
 552         signal(SIGUSR1, hand_down);
 553
 554         close(control[1]);
 555         close(output[1]);
 556
 557         /* We grab output so we can display it; we grab writes so we
 558          * can compare. */
 559         do {
 560                 struct pollfd pfd[2];
 561                 int ret;
 562
 563                 pfd[0].fd = output[0];
 564                 pfd[0].events = POLLIN|POLLHUP;
 565                 pfd[1].fd = control[0];
 566                 pfd[1].events = POLLIN|POLLHUP;
 567
 568                 if (type == SUCCESS)
 569                         ret = poll(pfd, 1, failtest_timeout_ms);
 570                 else
 571                         ret = poll(pfd, 2, failtest_timeout_ms);
 572
 573                 if (ret == 0)
 574                         hand_down(SIGUSR1);
 575                 if (ret < 0) {
 576                         if (errno == EINTR)
 577                                 continue;
 578                         err(1, "Poll returned %i", ret);
 579                 }
 580
 581                 if (pfd[0].revents & POLLIN) {
 582                         ssize_t len;
 583
 584                         out = realloc(out, outlen + 8192);
 585                         len = read(output[0], out + outlen, 8192);
 586                         outlen += len;
 587                 } else if (type != SUCCESS && (pfd[1].revents & POLLIN)) {
 588                         if (read_all(control[0], &type, sizeof(type))) {
 589                                 if (type == WRITE) {
 590                                         if (!read_write_info(control[0]))
 591                                                 break;
 592                                 } else if (type == RELEASE_LOCKS) {
 593                                         release_locks();
 594                                         /* FIXME: Tell them we're done... */
 595                                 }
 596                         }
 597                 } else if (pfd[0].revents & POLLHUP) {
 598                         break;
 599                 }
 600         } while (type != FAILURE);
 601
 602         close(output[0]);
 603         close(control[0]);
 604         waitpid(child, &status, 0);
 605         if (!WIFEXITED(status)) {
 606                 if (WTERMSIG(status) == SIGUSR1)
 607                         child_fail(out, outlen, "Timed out");
 608                 else
 609                         child_fail(out, outlen, "Killed by signal %u: ",
 610                                    WTERMSIG(status));
 611         }
 612         /* Child printed failure already, just pass up exit code. */
 613         if (type == FAILURE) {
 614                 fprintf(stderr, "%.*s", (int)outlen, out);
 615                 tell_parent(type);
 616                 exit(WEXITSTATUS(status) ? WEXITSTATUS(status) : 1);
 617         }
 618         if (WEXITSTATUS(status) != 0)
 619                 child_fail(out, outlen, "Exited with status %i: ",
 620                            WEXITSTATUS(status));
 621
 622         free(out);
 623         signal(SIGUSR1, SIG_DFL);
 624
 625         restore_files(files);
 626
 627         /* We continue onwards without failing. */
 628         call->fail = false;
 629         return false;
 630 }
 631
 632 static void cleanup_calloc(struct calloc_call *call)
 633 {
 634         free(call->ret);
 635 }
 636
 637 void *failtest_calloc(size_t nmemb, size_t size,
 638                       const char *file, unsigned line)
 639 {
 640         struct failtest_call *p;
 641         struct calloc_call call;
 642         call.nmemb = nmemb;
 643         call.size = size;
 644         p = add_history(FAILTEST_CALLOC, file, line, &call);
 645
 646         if (should_fail(p)) {
 647                 p->u.calloc.ret = NULL;
 648                 p->error = ENOMEM;
 649         } else {
 650                 p->u.calloc.ret = calloc(nmemb, size);
 651                 set_cleanup(p, cleanup_calloc, struct calloc_call);
 652         }
 653         errno = p->error;
 654         return p->u.calloc.ret;
 655 }
 656
 657 static void cleanup_malloc(struct malloc_call *call)
 658 {
 659         free(call->ret);
 660 }
 661
 662 void *failtest_malloc(size_t size, const char *file, unsigned line)
 663 {
 664         struct failtest_call *p;
 665         struct malloc_call call;
 666         call.size = size;
 667
 668         p = add_history(FAILTEST_MALLOC, file, line, &call);
 669         if (should_fail(p)) {
 670                 p->u.malloc.ret = NULL;
 671                 p->error = ENOMEM;
 672         } else {
 673                 p->u.malloc.ret = malloc(size);
 674                 set_cleanup(p, cleanup_malloc, struct malloc_call);
 675         }
 676         errno = p->error;
 677         return p->u.malloc.ret;
 678 }
 679
 680 static void cleanup_realloc(struct realloc_call *call)
 681 {
 682         free(call->ret);
 683 }
 684
 685 /* Walk back and find out if we got this ptr from a previous routine. */
 686 static void fixup_ptr_history(void *ptr)
 687 {
 688         struct failtest_call *i;
 689
 690         /* Start at end of history, work back. */
 691         tlist_for_each_rev(&history, i, list) {
 692                 switch (i->type) {
 693                 case FAILTEST_REALLOC:
 694                         if (i->u.realloc.ret == ptr) {
 695                                 i->cleanup = NULL;
 696                                 return;
 697                         }
 698                         break;
 699                 case FAILTEST_MALLOC:
 700                         if (i->u.malloc.ret == ptr) {
 701                                 i->cleanup = NULL;
 702                                 return;
 703                         }
 704                         break;
 705                 case FAILTEST_CALLOC:
 706                         if (i->u.calloc.ret == ptr) {
 707                                 i->cleanup = NULL;
 708                                 return;
 709                         }
 710                         break;
 711                 default:
 712                         break;
 713                 }
 714         }
 715 }
 716
 717 void *failtest_realloc(void *ptr, size_t size, const char *file, unsigned line)
 718 {
 719         struct failtest_call *p;
 720         struct realloc_call call;
 721         call.size = size;
 722         p = add_history(FAILTEST_REALLOC, file, line, &call);
 723
 724         /* FIXME: Try one child moving allocation, one not. */
 725         if (should_fail(p)) {
 726                 p->u.realloc.ret = NULL;
 727                 p->error = ENOMEM;
 728         } else {
 729                 /* Don't catch this one in the history fixup... */
 730                 p->u.realloc.ret = NULL;
 731                 fixup_ptr_history(ptr);
 732                 p->u.realloc.ret = realloc(ptr, size);
 733                 set_cleanup(p, cleanup_realloc, struct realloc_call);
 734         }
 735         errno = p->error;
 736         return p->u.realloc.ret;
 737 }
 738
 739 void failtest_free(void *ptr)
 740 {
 741         fixup_ptr_history(ptr);
 742         free(ptr);
 743 }
 744
 745 static void cleanup_open(struct open_call *call)
 746 {
 747         close(call->ret);
 748 }
 749
 750 int failtest_open(const char *pathname,
 751                   const char *file, unsigned line, ...)
 752 {
 753         struct failtest_call *p;
 754         struct open_call call;
 755         va_list ap;
 756
 757         call.pathname = strdup(pathname);
 758         va_start(ap, line);
 759         call.flags = va_arg(ap, int);
 760         if (call.flags & O_CREAT) {
 761                 call.mode = va_arg(ap, int);
 762                 va_end(ap);
 763         }
 764         p = add_history(FAILTEST_OPEN, file, line, &call);
 765         /* Avoid memory leak! */
 766         if (p == &unrecorded_call)
 767                 free((char *)call.pathname);
 768         p->u.open.ret = open(pathname, call.flags, call.mode);
 769
 770         if (p->u.open.ret == -1) {
 771                 p->fail = false;
 772                 p->error = errno;
 773         } else if (should_fail(p)) {
 774                 close(p->u.open.ret);
 775                 p->u.open.ret = -1;
 776                 /* FIXME: Play with error codes? */
 777                 p->error = EACCES;
 778         } else {
 779                 set_cleanup(p, cleanup_open, struct open_call);
 780         }
 781         errno = p->error;
 782         return p->u.open.ret;
 783 }
 784
 785 static void cleanup_pipe(struct pipe_call *call)
 786 {
 787         if (!call->closed[0])
 788                 close(call->fds[0]);
 789         if (!call->closed[1])
 790                 close(call->fds[1]);
 791 }
 792
 793 int failtest_pipe(int pipefd[2], const char *file, unsigned line)
 794 {
 795         struct failtest_call *p;
 796         struct pipe_call call;
 797
 798         p = add_history(FAILTEST_PIPE, file, line, &call);
 799         if (should_fail(p)) {
 800                 p->u.open.ret = -1;
 801                 /* FIXME: Play with error codes? */
 802                 p->error = EMFILE;
 803         } else {
 804                 p->u.pipe.ret = pipe(p->u.pipe.fds);
 805                 p->u.pipe.closed[0] = p->u.pipe.closed[1] = false;
 806                 set_cleanup(p, cleanup_pipe, struct pipe_call);
 807         }
 808         /* This causes valgrind to notice if they use pipefd[] after failure */
 809         memcpy(pipefd, p->u.pipe.fds, sizeof(p->u.pipe.fds));
 810         errno = p->error;
 811         return p->u.pipe.ret;
 812 }
 813
 814 ssize_t failtest_pread(int fd, void *buf, size_t count, off_t off,
 815                        const char *file, unsigned line)
 816 {
 817         struct failtest_call *p;
 818         struct read_call call;
 819         call.fd = fd;
 820         call.buf = buf;
 821         call.count = count;
 822         call.off = off;
 823         p = add_history(FAILTEST_READ, file, line, &call);
 824
 825         /* FIXME: Try partial read returns. */
 826         if (should_fail(p)) {
 827                 p->u.read.ret = -1;
 828                 p->error = EIO;
 829         } else {
 830                 p->u.read.ret = pread(fd, buf, count, off);
 831         }
 832         errno = p->error;
 833         return p->u.read.ret;
 834 }
 835
 836 ssize_t failtest_pwrite(int fd, const void *buf, size_t count, off_t off,
 837                         const char *file, unsigned line)
 838 {
 839         struct failtest_call *p;
 840         struct write_call call;
 841
 842         call.fd = fd;
 843         call.buf = buf;
 844         call.count = count;
 845         call.off = off;
 846         p = add_history(FAILTEST_WRITE, file, line, &call);
 847
 848         /* If we're a child, we need to make sure we write the same thing
 849          * to non-files as the parent does, so tell it. */
 850         if (control_fd != -1 && off == (off_t)-1) {
 851                 enum info_type type = WRITE;
 852
 853                 write_all(control_fd, &type, sizeof(type));
 854                 write_all(control_fd, &p->u.write, sizeof(p->u.write));
 855                 write_all(control_fd, buf, count);
 856         }
 857
 858         /* FIXME: Try partial write returns. */
 859         if (should_fail(p)) {
 860                 p->u.write.ret = -1;
 861                 p->error = EIO;
 862         } else {
 863                 /* FIXME: We assume same write order in parent and child */
 864                 if (off == (off_t)-1 && child_writes_num != 0) {
 865                         if (child_writes[0].fd != fd)
 866                                 errx(1, "Child wrote to fd %u, not %u?",
 867                                      child_writes[0].fd, fd);
 868                         if (child_writes[0].off != p->u.write.off)
 869                                 errx(1, "Child wrote to offset %zu, not %zu?",
 870                                      (size_t)child_writes[0].off,
 871                                      (size_t)p->u.write.off);
 872                         if (child_writes[0].count != count)
 873                                 errx(1, "Child wrote length %zu, not %zu?",
 874                                      child_writes[0].count, count);
 875                         if (memcmp(child_writes[0].buf, buf, count)) {
 876                                 child_fail(NULL, 0,
 877                                            "Child wrote differently to"
 878                                            " fd %u than we did!\n", fd);
 879                         }
 880                         free((char *)child_writes[0].buf);
 881                         child_writes_num--;
 882                         memmove(&child_writes[0], &child_writes[1],
 883                                 sizeof(child_writes[0]) * child_writes_num);
 884
 885                         /* Is this is a socket or pipe, child wrote it
 886                            already. */
 887                         if (p->u.write.off == (off_t)-1) {
 888                                 p->u.write.ret = count;
 889                                 errno = p->error;
 890                                 return p->u.write.ret;
 891                         }
 892                 }
 893                 p->u.write.ret = pwrite(fd, buf, count, off);
 894         }
 895         errno = p->error;
 896         return p->u.write.ret;
 897 }
 898
 899 ssize_t failtest_read(int fd, void *buf, size_t count,
 900                       const char *file, unsigned line)
 901 {
 902         return failtest_pread(fd, buf, count, lseek(fd, 0, SEEK_CUR),
 903                               file, line);
 904 }
 905
 906 ssize_t failtest_write(int fd, const void *buf, size_t count,
 907                        const char *file, unsigned line)
 908 {
 909         return failtest_pwrite(fd, buf, count, lseek(fd, 0, SEEK_CUR),
 910                                file, line);
 911 }
 912
 913 static struct lock_info *WARN_UNUSED_RESULT
 914 add_lock(struct lock_info *locks, int fd, off_t start, off_t end, int type)
 915 {
 916         unsigned int i;
 917         struct lock_info *l;
 918
 919         for (i = 0; i < lock_num; i++) {
 920                 l = &locks[i];
 921
 922                 if (l->fd != fd)
 923                         continue;
 924                 /* Four cases we care about:
 925                  * Start overlap:
 926                  *      l =    |      |
 927                  *      new = |   |
 928                  * Mid overlap:
 929                  *      l =    |      |
 930                  *      new =    |  |
 931                  * End overlap:
 932                  *      l =    |      |
 933                  *      new =      |    |
 934                  * Total overlap:
 935                  *      l =    |      |
 936                  *      new = |         |
 937                  */
 938                 if (start > l->start && end < l->end) {
 939                         /* Mid overlap: trim entry, add new one. */
 940                         off_t new_start, new_end;
 941                         new_start = end + 1;
 942                         new_end = l->end;
 943                         l->end = start - 1;
 944                         locks = add_lock(locks,
 945                                          fd, new_start, new_end, l->type);
 946                         l = &locks[i];
 947                 } else if (start <= l->start && end >= l->end) {
 948                         /* Total overlap: eliminate entry. */
 949                         l->end = 0;
 950                         l->start = 1;
 951                 } else if (end >= l->start && end < l->end) {
 952                         /* Start overlap: trim entry. */
 953                         l->start = end + 1;
 954                 } else if (start > l->start && start <= l->end) {
 955                         /* End overlap: trim entry. */
 956                         l->end = start-1;
 957                 }
 958                 /* Nothing left?  Remove it. */
 959                 if (l->end < l->start) {
 960                         memmove(l, l + 1, (--lock_num - i) * sizeof(l[0]));
 961                         i--;
 962                 }
 963         }
 964
 965         if (type != F_UNLCK) {
 966                 locks = realloc(locks, (lock_num + 1) * sizeof(*locks));
 967                 l = &locks[lock_num++];
 968                 l->fd = fd;
 969                 l->start = start;
 970                 l->end = end;
 971                 l->type = type;
 972         }
 973         return locks;
 974 }
 975
 976 /* We trap this so we can record it: we don't fail it. */
 977 int failtest_close(int fd, const char *file, unsigned line)
 978 {
 979         struct failtest_call *i;
 980         struct close_call call;
 981         struct failtest_call *p;
 982
 983         call.fd = fd;
 984         p = add_history(FAILTEST_CLOSE, file, line, &call);
 985         p->fail = false;
 986
 987         /* Consume close from failpath. */
 988         if (failpath)
 989                 if (should_fail(p))
 990                         abort();
 991
 992         if (fd < 0)
 993                 return close(fd);
 994
 995         /* Trace history to find source of fd. */
 996         tlist_for_each_rev(&history, i, list) {
 997                 switch (i->type) {
 998                 case FAILTEST_PIPE:
 999                         /* From a pipe? */
1000                         if (i->u.pipe.fds[0] == fd) {
1001                                 assert(!i->u.pipe.closed[0]);
1002                                 i->u.pipe.closed[0] = true;
1003                                 if (i->u.pipe.closed[1])
1004                                         i->cleanup = NULL;
1005                                 goto out;
1006                         }
1007                         if (i->u.pipe.fds[1] == fd) {
1008                                 assert(!i->u.pipe.closed[1]);
1009                                 i->u.pipe.closed[1] = true;
1010                                 if (i->u.pipe.closed[0])
1011                                         i->cleanup = NULL;
1012                                 goto out;
1013                         }
1014                         break;
1015                 case FAILTEST_OPEN:
1016                         if (i->u.open.ret == fd) {
1017                                 assert((void *)i->cleanup
1018                                        == (void *)cleanup_open);
1019                                 i->cleanup = NULL;
1020                                 goto out;
1021                         }
1022                         break;
1023                 default:
1024                         break;
1025                 }
1026         }
1027
1028 out:
1029         locks = add_lock(locks, fd, 0, off_max(), F_UNLCK);
1030         return close(fd);
1031 }
1032
1033 /* Zero length means "to end of file" */
1034 static off_t end_of(off_t start, off_t len)
1035 {
1036         if (len == 0)
1037                 return off_max();
1038         return start + len - 1;
1039 }
1040
1041 /* FIXME: This only handles locks, really. */
1042 int failtest_fcntl(int fd, const char *file, unsigned line, int cmd, ...)
1043 {
1044         struct failtest_call *p;
1045         struct fcntl_call call;
1046         va_list ap;
1047
1048         call.fd = fd;
1049         call.cmd = cmd;
1050
1051         /* Argument extraction. */
1052         switch (cmd) {
1053         case F_SETFL:
1054         case F_SETFD:
1055                 va_start(ap, cmd);
1056                 call.arg.l = va_arg(ap, long);
1057                 va_end(ap);
1058                 return fcntl(fd, cmd, call.arg.l);
1059         case F_GETFD:
1060         case F_GETFL:
1061                 return fcntl(fd, cmd);
1062         case F_GETLK:
1063                 get_locks();
1064                 va_start(ap, cmd);
1065                 call.arg.fl = *va_arg(ap, struct flock *);
1066                 va_end(ap);
1067                 return fcntl(fd, cmd, &call.arg.fl);
1068         case F_SETLK:
1069         case F_SETLKW:
1070                 va_start(ap, cmd);
1071                 call.arg.fl = *va_arg(ap, struct flock *);
1072                 va_end(ap);
1073                 break;
1074         default:
1075                 /* This means you need to implement it here. */
1076                 err(1, "failtest: unknown fcntl %u", cmd);
1077         }
1078
1079         p = add_history(FAILTEST_FCNTL, file, line, &call);
1080
1081         if (should_fail(p)) {
1082                 p->u.fcntl.ret = -1;
1083                 if (p->u.fcntl.cmd == F_SETLK)
1084                         p->error = EAGAIN;
1085                 else
1086                         p->error = EDEADLK;
1087         } else {
1088                 get_locks();
1089                 p->u.fcntl.ret = fcntl(p->u.fcntl.fd, p->u.fcntl.cmd,
1090                                        &p->u.fcntl.arg.fl);
1091                 if (p->u.fcntl.ret == -1)
1092                         p->error = errno;
1093                 else {
1094                         /* We don't handle anything else yet. */
1095                         assert(p->u.fcntl.arg.fl.l_whence == SEEK_SET);
1096                         locks = add_lock(locks,
1097                                          p->u.fcntl.fd,
1098                                          p->u.fcntl.arg.fl.l_start,
1099                                          end_of(p->u.fcntl.arg.fl.l_start,
1100                                                 p->u.fcntl.arg.fl.l_len),
1101                                          p->u.fcntl.arg.fl.l_type);
1102                 }
1103         }
1104         errno = p->error;
1105         return p->u.fcntl.ret;
1106 }
1107
1108 pid_t failtest_getpid(const char *file, unsigned line)
1109 {
1110         /* You must call failtest_init first! */
1111         assert(orig_pid);
1112         return orig_pid;
1113 }
1114
1115 void failtest_init(int argc, char *argv[])
1116 {
1117         unsigned int i;
1118
1119         orig_pid = getpid();
1120
1121         warnfd = move_fd_to_high(dup(STDERR_FILENO));
1122         for (i = 1; i < argc; i++) {
1123                 if (!strncmp(argv[i], "--failpath=", strlen("--failpath="))) {
1124                         failpath = argv[i] + strlen("--failpath=");
1125                 } else if (strcmp(argv[i], "--tracepath") == 0) {
1126                         tracefd = warnfd;
1127                         failtest_timeout_ms = -1;
1128                 } else if (!strncmp(argv[i], "--debugpath=",
1129                                     strlen("--debugpath="))) {
1130                         debugpath = argv[i] + strlen("--debugpath=");
1131                 }
1132         }
1133         start = time_now();
1134 }
1135
1136 bool failtest_has_failed(void)
1137 {
1138         return control_fd != -1;
1139 }
1140
1141 void failtest_exit(int status)
1142 {
1143         if (failtest_exit_check) {
1144                 if (!failtest_exit_check(&history))
1145                         child_fail(NULL, 0, "failtest_exit_check failed\n");
1146         }
1147
1148         failtest_cleanup(false, status);
1149 }