ccanlint: use valgrind options when debugging too
[ccan] / tools / ccanlint / tests / run_tests_valgrind.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str/str.h>
5 #include <ccan/foreach/foreach.h>
6 #include <ccan/grab_file/grab_file.h>
7 #include <ccan/str_talloc/str_talloc.h>
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <fcntl.h>
11 #include <unistd.h>
12 #include <limits.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <err.h>
17 #include <string.h>
18 #include <ctype.h>
19
20 /* Note: we already test safe_mode in run_tests.c */
21 static const char *can_run_vg(struct manifest *m)
22 {
23         unsigned int timeleft = default_timeout_ms;
24         char *output;
25
26         if (!run_command(m, &timeleft, &output,
27                          "valgrind -q --error-exitcode=0 true"))
28                 return talloc_asprintf(m, "No valgrind support: %s", output);
29         return NULL;
30 }
31
32 /* Example output:
33 ==2749== Conditional jump or move depends on uninitialised value(s)
34 ==2749==    at 0x4026C60: strnlen (mc_replace_strmem.c:263)
35 ==2749==    by 0x40850E3: vfprintf (vfprintf.c:1614)
36 ==2749==    by 0x408EACF: printf (printf.c:35)
37 ==2749==    by 0x8048465: main (in /tmp/foo)
38 ==2749== 
39 ==2749== 1 bytes in 1 blocks are definitely lost in loss record 1 of 1
40 ==2749==    at 0x4025BD3: malloc (vg_replace_malloc.c:236)
41 ==2749==    by 0x8048444: main (in /tmp/foo)
42 ==2749== 
43 */
44
45 static bool blank_line(const char *line)
46 {
47         return line[strspn(line, "=0123456789 ")] == '\0';
48 }
49
50 static char *get_leaks(const char *output, char **errs)
51 {
52         char *leaks = talloc_strdup(output, "");
53         unsigned int i, num;
54         char **lines = strsplit(output, output, "\n", &num);
55
56         *errs = talloc_strdup(output, "");
57         for (i = 0; i < num; i++) {
58                 if (strstr(lines[i], " lost ")) {
59                         /* A leak... */
60                         if (strstr(lines[i], " definitely lost ")) {
61                                 /* Definite leak, report. */
62                                 while (lines[i] && !blank_line(lines[i])) {
63                                         leaks = talloc_append_string(leaks,
64                                                                      lines[i]);
65                                         leaks = talloc_append_string(leaks,
66                                                                      "\n");
67                                         i++;
68                                 }
69                         } else
70                                 /* Not definite, ignore. */
71                                 while (lines[i] && !blank_line(lines[i]))
72                                         i++;
73                 } else {
74                         /* A real error. */
75                         while (lines[i] && !blank_line(lines[i])) {
76                                 *errs = talloc_append_string(*errs, lines[i]);
77                                 *errs = talloc_append_string(*errs, "\n");
78                                 i++;
79                         }
80                 }
81         }
82         if (!leaks[0]) {
83                 talloc_free(leaks);
84                 leaks = NULL;
85         }
86         if (!(*errs)[0]) {
87                 talloc_free(*errs);
88                 *errs = NULL;
89         }
90         return leaks;
91 }
92
93 /* FIXME: Run examples, too! */
94 static void do_run_tests_vg(struct manifest *m,
95                              bool keep,
96                             unsigned int *timeleft,
97                             struct score *score)
98 {
99         struct ccan_file *i;
100         struct list_head *list;
101         char *cmdout;
102
103         /* This is slow, so we run once but grab leak info. */
104         score->total = 0;
105         foreach_ptr(list, &m->run_tests, &m->api_tests) {
106                 list_for_each(list, i, list) {
107                         char *output, *err;
108                         score->total++;
109                         /* FIXME: Valgrind's output sucks.  XML is unreadable by
110                          * humans, and you can't have both. */
111                         if (run_command(score, timeleft, &cmdout,
112                                         "valgrind -q --leak-check=full"
113                                         " --log-fd=3 %s %s"
114                                         " 3> valgrind.log",
115                                         run_tests_vg.options ?
116                                         run_tests_vg.options : "",
117                                         i->compiled)) {
118                                 output = grab_file(i, "valgrind.log", NULL);
119                                 if (!output || output[0] == '\0') {
120                                         err = NULL;
121                                 } else {
122                                         i->leak_info = get_leaks(output, &err);
123                                 }
124                                 if (err) {
125                                         score_file_error(score, i, 0, err);
126                                 } else
127                                         score->score++;
128                         } else {
129                                 score_file_error(score, i, 0, cmdout);
130                         }
131                 }
132         }
133
134         if (score->score == score->total)
135                 score->pass = true;
136 }
137
138 static void do_leakcheck_vg(struct manifest *m,
139                             bool keep,
140                             unsigned int *timeleft,
141                             struct score *score)
142 {
143         struct ccan_file *i;
144         struct list_head *list;
145         bool leaks = false;
146
147         foreach_ptr(list, &m->run_tests, &m->api_tests) {
148                 list_for_each(list, i, list) {
149                         if (i->leak_info) {
150                                 score_file_error(score, i, 0, i->leak_info);
151                                 leaks = true;
152                         }
153                 }
154         }
155
156         if (!leaks) {
157                 score->score = 1;
158                 score->pass = true;
159         }
160 }
161
162 /* Gcc's warn_unused_result is fascist bullshit. */
163 #define doesnt_matter()
164
165 static void run_under_debugger_vg(struct manifest *m, struct score *score)
166 {
167         struct file_error *first;
168         char *command;
169
170         if (!ask("Should I run the first failing test under the debugger?"))
171                 return;
172
173         first = list_top(&score->per_file_errors, struct file_error, list);
174         command = talloc_asprintf(m, "valgrind --db-attach=yes%s %s",
175                                   run_tests_vg.options ?
176                                   run_tests_vg.options : "",
177                                   first->file->compiled);
178         if (system(command))
179                 doesnt_matter();
180 }
181
182 struct ccanlint run_tests_vg = {
183         .key = "valgrind-tests",
184         .name = "Module's run and api tests succeed under valgrind",
185         .can_run = can_run_vg,
186         .check = do_run_tests_vg,
187         .handle = run_under_debugger_vg,
188         .takes_options = true
189 };
190
191 REGISTER_TEST(run_tests_vg, &run_tests, NULL);
192
193 struct ccanlint run_tests_vg_leak = {
194         .key = "valgrind-leaks",
195         .name = "Module's run and api tests leak memory",
196         .check = do_leakcheck_vg,
197 };
198
199 REGISTER_TEST(run_tests_vg_leak, &run_tests_vg, NULL);