failtest: hook can return FAIL_PROBE

tdb2 has various places where it recovers from failure (eg. falling
back when it can't open /dev/urandom, or allocation for error
logging).  We want to test those paths, but doing so thoroughly causes
cominatorial explosion.

Add FAIL_PROBE for such cases: in this case it goes only 3 more calls
deep.

diff --git a/ccan/failtest/failtest.c b/ccan/failtest/failtest.c
index 9ecc535a3fb73ac736f7474e88508c950fba1345..9972db3cd29fc8ad83a9d3c51bc8fedf65ef4a76 100644 (file)
--- a/ccan/failtest/failtest.c
+++ b/ccan/failtest/failtest.c
@@ -17,8 +17,7 @@
 #include <ccan/failtest/failtest.h>
 #include <ccan/build_assert/build_assert.h>
 
-bool (*failtest_hook)(struct failtest_call *history, unsigned num)
-= failtest_default_hook;
+enum failtest_result (*failtest_hook)(struct failtest_call *, unsigned);
 
 static int tracefd = -1;
 
@@ -48,6 +47,7 @@ static struct failtest_call *history = NULL;
 static unsigned int history_num = 0;
 static int control_fd = -1;
 static struct timeval start;
+static unsigned int probe_count = 0;
 
 static struct write_call *child_writes = NULL;
 static unsigned int child_writes_num = 0;
@@ -86,11 +86,6 @@ static struct failtest_call *add_history_(enum failtest_call_type type,
 #define set_cleanup(call, clean, type)                 \
        (call)->cleanup = (void *)((void)sizeof(clean((type *)NULL)), (clean))
 
-bool failtest_default_hook(struct failtest_call *history, unsigned num)
-{
-       return true;
-}
-
 static bool read_write_info(int fd)
 {
        struct write_call *w;
@@ -305,6 +300,40 @@ static void restore_files(struct saved_file *s)
        }
 }
 
+/* Free up memory, so valgrind doesn't report leaks. */
+static void free_everything(void)
+{
+       unsigned int i;
+
+       /* We don't do this in cleanup: needed even for failed opens. */
+       for (i = 0; i < history_num; i++) {
+               if (history[i].type == FAILTEST_OPEN)
+                       free((char *)history[i].u.open.pathname);
+       }
+       free(history);
+}
+
+static NORETURN void failtest_cleanup(bool forced_cleanup, int status)
+{
+       int i;
+
+       if (forced_cleanup)
+               history_num--;
+
+       /* Cleanup everything, in reverse order. */
+       for (i = history_num - 1; i >= 0; i--)
+               if (history[i].cleanup)
+                       history[i].cleanup(&history[i].u);
+
+       free_everything();
+
+       if (control_fd == -1)
+               exit(status);
+
+       tell_parent(SUCCESS);
+       exit(0);
+}
+
 static bool should_fail(struct failtest_call *call)
 {
        int status;
@@ -314,6 +343,10 @@ static bool should_fail(struct failtest_call *call)
        size_t outlen = 0;
        struct saved_file *files;
 
+       /* Are we probing? */
+       if (probe_count && --probe_count == 0)
+               failtest_cleanup(true, 0);
+
        if (call == &unrecorded_call)
                return false;
 
@@ -353,9 +386,24 @@ static bool should_fail(struct failtest_call *call)
                }
        }
 
-       if (!failtest_hook(history, history_num)) {
-               call->fail = false;
-               return false;
+       if (failtest_hook) {
+               switch (failtest_hook(history, history_num)) {
+               case FAIL_OK:
+                       break;
+               case FAIL_DONT_FAIL:
+                       call->fail = false;
+                       return false;
+               case FAIL_PROBE:
+                       /* Already down probe path?  Stop now. */
+                       if (probe_count)
+                               failtest_cleanup(true, 0);
+                       /* FIXME: We should run *parent* and run probe until
+                        * calls match up again. */
+                       probe_count = 3;
+                       break;
+               default:
+                       abort();
+               }
        }
 
        files = save_files();
@@ -974,39 +1022,12 @@ void failtest_init(int argc, char *argv[])
        gettimeofday(&start, NULL);
 }
 
-/* Free up memory, so valgrind doesn't report leaks. */
-static void free_everything(void)
-{
-       unsigned int i;
-
-       /* We don't do this in cleanup: needed even for failed opens. */
-       for (i = 0; i < history_num; i++) {
-               if (history[i].type == FAILTEST_OPEN)
-                       free((char *)history[i].u.open.pathname);
-       }
-       free(history);
-}
-
 void failtest_exit(int status)
 {
-       int i;
-
        if (failtest_exit_check) {
                if (!failtest_exit_check(history, history_num))
                        child_fail(NULL, 0, "failtest_exit_check failed\n");
        }
 
-       if (control_fd == -1) {
-               free_everything();
-               exit(status);
-       }
-
-       /* Cleanup everything, in reverse order. */
-       for (i = history_num - 1; i >= 0; i--)
-               if (history[i].cleanup)
-                       history[i].cleanup(&history[i].u);
-
-       free_everything();
-       tell_parent(SUCCESS);
-       exit(0);
+       failtest_cleanup(false, status);
 }

diff --git a/ccan/failtest/failtest.h b/ccan/failtest/failtest.h
index ec238af0ab6f7d195de07df7e597c45c54a61145..1aa921813bf7af6c433ab1f6e46ebfe6032b91b7 100644 (file)
--- a/ccan/failtest/failtest.h
+++ b/ccan/failtest/failtest.h
@@ -143,30 +143,42 @@ struct failtest_call {
        } u;
 };
 
+enum failtest_result {
+       /* Yes try failing this call. */
+       FAIL_OK,
+       /* No, don't try failing this call. */
+       FAIL_DONT_FAIL,
+       /* Try failing this call but don't go too far down that path. */
+       FAIL_PROBE,
+};
+
 /**
  * failtest_hook - whether a certain call should fail or not.
  * @history: the ordered history of all failtest calls.
  * @num: the number of elements in @history (greater than 0)
  *
  * The default value of this hook is failtest_default_hook(), which returns
- * true (ie. yes, fail the call).
+ * FAIL_OK (ie. yes, fail the call).
  *
  * You can override it, and avoid failing certain calls.  The parameters
  * of the call (but not the return value(s)) will be filled in for the last
  * call.
  *
  * Example:
- *     static bool dont_fail_allocations(struct failtest_call *history,
- *                                       unsigned num)
+ *     static enum failtest_result dont_fail_alloc(struct failtest_call *hist,
+ *                                                 unsigned num)
  *     {
- *             return history[num-1].type != FAILTEST_MALLOC
- *                     && history[num-1].type != FAILTEST_CALLOC
- *                     && history[num-1].type != FAILTEST_REALLOC;
+ *             if (hist[num-1].type == FAILTEST_MALLOC
+ *                     || hist[num-1].type == FAILTEST_CALLOC
+ *                     || hist[num-1].type == FAILTEST_REALLOC)
+ *                     return FAIL_DONT_FAIL;
+ *             return FAIL_OK;
  *     }
  *     ...
- *             failtest_hook = dont_fail_allocations;
+ *             failtest_hook = dont_fail_alloc;
  */
-extern bool (*failtest_hook)(struct failtest_call *history, unsigned num);
+extern enum failtest_result
+(*failtest_hook)(struct failtest_call *history, unsigned num);
 
 /**
  * failtest_exit_check - hook for additional checks on a failed child.
@@ -183,9 +195,6 @@ extern bool (*failtest_hook)(struct failtest_call *history, unsigned num);
 extern bool (*failtest_exit_check)(struct failtest_call *history,
                                   unsigned num);
 
-/* This usually fails the call. */
-bool failtest_default_hook(struct failtest_call *history, unsigned num);
-
 /**
  * failtest_timeout_ms - how long to wait before killing child.
  *

diff --git a/ccan/failtest/test/run-locking.c b/ccan/failtest/test/run-locking.c
index b25cfab9a29b36793359cda9c991f772b6a2c22c..6ebcd6ea504017b6de1374ed40020545c02077d5 100644 (file)
--- a/ccan/failtest/test/run-locking.c
+++ b/ccan/failtest/test/run-locking.c
@@ -10,9 +10,10 @@
 #define SIZE 8
 
 /* We don't want to fork and fail; we're just testing lock recording. */
-static bool dont_fail(struct failtest_call *history, unsigned num)
+static enum failtest_result dont_fail(struct failtest_call *history,
+                                     unsigned num)
 {
-       return false;
+       return FAIL_DONT_FAIL;
 }
 
 static bool place_lock(int fd, char lockarr[], unsigned pos, unsigned size,