ccanlint: have valgrind fail with an error, always
[ccan] / tools / ccanlint / tests / run_tests_valgrind.c
index 6312a0a864327988c53b39f080eb0b6daedcaecc..ec3a31981c21f01331fd6d1a934a95f33f452f60 100644 (file)
@@ -2,6 +2,9 @@
 #include <tools/tools.h>
 #include <ccan/talloc/talloc.h>
 #include <ccan/str/str.h>
+#include <ccan/foreach/foreach.h>
+#include <ccan/grab_file/grab_file.h>
+#include <ccan/str_talloc/str_talloc.h>
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <fcntl.h>
 /* Note: we already test safe_mode in run_tests.c */
 static const char *can_run_vg(struct manifest *m)
 {
-       char *output = run_command(m, "valgrind -q true");
+       unsigned int timeleft = default_timeout_ms;
+       char *output;
 
-       if (output)
+       if (!run_command(m, &timeleft, &output,
+                        "valgrind -q --error-exitcode=0 true"))
                return talloc_asprintf(m, "No valgrind support: %s", output);
        return NULL;
 }
 
-struct run_tests_result {
-       struct list_node list;
-       struct ccan_file *file;
-       const char *output;
-};
+/* Example output:
+==2749== Conditional jump or move depends on uninitialised value(s)
+==2749==    at 0x4026C60: strnlen (mc_replace_strmem.c:263)
+==2749==    by 0x40850E3: vfprintf (vfprintf.c:1614)
+==2749==    by 0x408EACF: printf (printf.c:35)
+==2749==    by 0x8048465: main (in /tmp/foo)
+==2749== 
+==2749== 1 bytes in 1 blocks are definitely lost in loss record 1 of 1
+==2749==    at 0x4025BD3: malloc (vg_replace_malloc.c:236)
+==2749==    by 0x8048444: main (in /tmp/foo)
+==2749== 
+*/
+
+static bool blank_line(const char *line)
+{
+       return line[strspn(line, "=0123456789 ")] == '\0';
+}
 
-static void *do_run_tests_vg(struct manifest *m)
+static char *get_leaks(const char *output, char **errs)
 {
-       struct list_head *list = talloc(m, struct list_head);
-       struct run_tests_result *res;
-       struct ccan_file *i;
-       char *cmdout;
-       char *olddir;
-
-       /* We run tests in the module directory, so any paths
-        * referenced can all be module-local. */
-       olddir = talloc_getcwd(m);
-       if (!olddir)
-               err(1, "Could not save cwd");
-       if (chdir(m->dir) != 0)
-               err(1, "Could not chdir to %s", m->dir);
-
-       list_head_init(list);
-
-       list_for_each(&m->run_tests, i, list) {
-               run_tests_vg.total_score++;
-               /* FIXME: timeout here */
-               cmdout = run_command(m, "valgrind -q %s", i->compiled);
-               if (cmdout) {
-                       res = talloc(list, struct run_tests_result);
-                       res->file = i;
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
+       char *leaks = talloc_strdup(output, "");
+       unsigned int i, num;
+       char **lines = strsplit(output, output, "\n", &num);
+
+       *errs = talloc_strdup(output, "");
+       for (i = 0; i < num; i++) {
+               if (strstr(lines[i], " lost ")) {
+                       /* A leak... */
+                       if (strstr(lines[i], " definitely lost ")) {
+                               /* Definite leak, report. */
+                               while (lines[i] && !blank_line(lines[i])) {
+                                       leaks = talloc_append_string(leaks,
+                                                                    lines[i]);
+                                       leaks = talloc_append_string(leaks,
+                                                                    "\n");
+                                       i++;
+                               }
+                       } else
+                               /* Not definite, ignore. */
+                               while (lines[i] && !blank_line(lines[i]))
+                                       i++;
+               } else {
+                       /* A real error. */
+                       while (lines[i] && !blank_line(lines[i])) {
+                               *errs = talloc_append_string(*errs, lines[i]);
+                               *errs = talloc_append_string(*errs, "\n");
+                               i++;
+                       }
                }
        }
-
-       list_for_each(&m->api_tests, i, list) {
-               run_tests_vg.total_score++;
-               /* FIXME: timeout here */
-               cmdout = run_command(m, "valgrind -q %s", i->compiled);
-               if (cmdout) {
-                       res = talloc(list, struct run_tests_result);
-                       res->file = i;
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
+       if (!leaks[0]) {
+               talloc_free(leaks);
+               leaks = NULL;
        }
-
-       if (list_empty(list)) {
-               talloc_free(list);
-               list = NULL;
+       if (!(*errs)[0]) {
+               talloc_free(*errs);
+               *errs = NULL;
        }
-
-       if (chdir(olddir) != 0)
-               err(1, "Could not chdir to %s", olddir);
-
-       return list;
+       return leaks;
 }
 
-static unsigned int score_run_tests_vg(struct manifest *m, void *check_result)
+/* FIXME: Run examples, too! */
+static void do_run_tests_vg(struct manifest *m,
+                            bool keep,
+                           unsigned int *timeleft,
+                           struct score *score)
 {
-       struct list_head *list = check_result;
-       struct run_tests_result *i;
-       unsigned int score = run_tests_vg.total_score;
+       struct ccan_file *i;
+       struct list_head *list;
+       char *cmdout;
 
-       list_for_each(list, i, list)
-               score--;
-       return score;
+       /* This is slow, so we run once but grab leak info. */
+       score->total = 0;
+       foreach_ptr(list, &m->run_tests, &m->api_tests) {
+               list_for_each(list, i, list) {
+                       char *output, *err;
+                       score->total++;
+                       /* FIXME: Valgrind's output sucks.  XML is unreadable by
+                        * humans, and you can't have both. */
+                       run_command(score, timeleft, &cmdout,
+                                   "valgrind -q --error-exitcode=101"
+                                   " --leak-check=full"
+                                   " --log-fd=3 %s %s"
+                                   " 3> valgrind.log",
+                                   run_tests_vg.options ?
+                                   run_tests_vg.options : "",
+                                   i->compiled);
+                       output = grab_file(i, "valgrind.log", NULL);
+                       if (!output || output[0] == '\0') {
+                               err = NULL;
+                       } else {
+                               i->leak_info = get_leaks(output, &err);
+                       }
+                       if (err)
+                               score_file_error(score, i, 0, err);
+                       else
+                               score->score++;
+               }
+       }
+
+       if (score->score == score->total)
+               score->pass = true;
 }
 
-static const char *describe_run_tests_vg(struct manifest *m,
-                                        void *check_result)
+static void do_leakcheck_vg(struct manifest *m,
+                           bool keep,
+                           unsigned int *timeleft,
+                           struct score *score)
 {
-       struct list_head *list = check_result;
-       char *descrip = talloc_strdup(check_result, "Running tests under valgrind failed:\n");
-       struct run_tests_result *i;
-
-       list_for_each(list, i, list)
-               descrip = talloc_asprintf_append(descrip, "Running %s:\n%s",
-                                                i->file->name, i->output);
-       return descrip;
+       struct ccan_file *i;
+       struct list_head *list;
+       bool leaks = false;
+
+       foreach_ptr(list, &m->run_tests, &m->api_tests) {
+               list_for_each(list, i, list) {
+                       if (i->leak_info) {
+                               score_file_error(score, i, 0, i->leak_info);
+                               leaks = true;
+                       }
+               }
+       }
+
+       if (!leaks) {
+               score->score = 1;
+               score->pass = true;
+       }
 }
 
 /* Gcc's warn_unused_result is fascist bullshit. */
 #define doesnt_matter()
 
-static void run_under_debugger_vg(struct manifest *m, void *check_result)
+static void run_under_debugger_vg(struct manifest *m, struct score *score)
 {
-       struct list_head *list = check_result;
-       struct run_tests_result *first;
+       struct file_error *first;
        char *command;
 
        if (!ask("Should I run the first failing test under the debugger?"))
                return;
 
-       first = list_top(list, struct run_tests_result, list);
-       command = talloc_asprintf(m, "valgrind --db-attach=yes %s",
+       first = list_top(&score->per_file_errors, struct file_error, list);
+       command = talloc_asprintf(m, "valgrind --db-attach=yes%s %s",
+                                 run_tests_vg.options ?
+                                 run_tests_vg.options : "",
                                  first->file->compiled);
        if (system(command))
                doesnt_matter();
 }
 
 struct ccanlint run_tests_vg = {
-       .key = "valgrind",
-       .name = "run and api tests under valgrind",
-       .score = score_run_tests_vg,
-       .check = do_run_tests_vg,
-       .describe = describe_run_tests_vg,
+       .key = "valgrind-tests",
+       .name = "Module's run and api tests succeed under valgrind",
        .can_run = can_run_vg,
-       .handle = run_under_debugger_vg
+       .check = do_run_tests_vg,
+       .handle = run_under_debugger_vg,
+       .takes_options = true
 };
 
 REGISTER_TEST(run_tests_vg, &run_tests, NULL);
+
+struct ccanlint run_tests_vg_leak = {
+       .key = "valgrind-leaks",
+       .name = "Module's run and api tests leak memory",
+       .check = do_leakcheck_vg,
+};
+
+REGISTER_TEST(run_tests_vg_leak, &run_tests_vg, NULL);