2a227c88743ead915aad21db6fd82f35e1bbb352
[ccan] / tools / ccanlint / tests / tests_pass_valgrind.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str/str.h>
5 #include <ccan/foreach/foreach.h>
6 #include <ccan/grab_file/grab_file.h>
7 #include <ccan/str_talloc/str_talloc.h>
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <fcntl.h>
11 #include <unistd.h>
12 #include <limits.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <err.h>
17 #include <string.h>
18 #include <ctype.h>
19
20 REGISTER_TEST(tests_pass_valgrind);
21 REGISTER_TEST(tests_pass_valgrind_noleaks);
22
23 /* Note: we already test safe_mode in run_tests.c */
24 static const char *can_run_vg(struct manifest *m)
25 {
26         unsigned int timeleft = default_timeout_ms;
27         char *output;
28
29         if (!run_command(m, &timeleft, &output,
30                          "valgrind -q --error-exitcode=0 true"))
31                 return talloc_asprintf(m, "No valgrind support: %s", output);
32         return NULL;
33 }
34
35 /* Example output:
36 ==2749== Conditional jump or move depends on uninitialised value(s)
37 ==2749==    at 0x4026C60: strnlen (mc_replace_strmem.c:263)
38 ==2749==    by 0x40850E3: vfprintf (vfprintf.c:1614)
39 ==2749==    by 0x408EACF: printf (printf.c:35)
40 ==2749==    by 0x8048465: main (in /tmp/foo)
41 ==2749== 
42 ==2749== 1 bytes in 1 blocks are definitely lost in loss record 1 of 1
43 ==2749==    at 0x4025BD3: malloc (vg_replace_malloc.c:236)
44 ==2749==    by 0x8048444: main (in /tmp/foo)
45 ==2749== 
46 */
47
48 static bool blank_line(const char *line)
49 {
50         return line[strspn(line, "=0123456789 ")] == '\0';
51 }
52
53 /* Removes matching lines from lines array, returns them.  FIXME: Hacky. */
54 static char **extract_matching(const char *prefix, char *lines[])
55 {
56         unsigned int i, num_ret = 0;
57         char **ret = talloc_array(lines, char *, talloc_array_length(lines));
58
59         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
60                 if (strstarts(lines[i], prefix)) {
61                         ret[num_ret++] = talloc_steal(ret, lines[i]);
62                         lines[i] = (char *)"";
63                 }
64         }
65         ret[num_ret++] = NULL;
66
67         /* Make sure length is correct! */
68         return talloc_realloc(NULL, ret, char *, num_ret);
69 }
70
71 static char *get_leaks(char *lines[], char **errs)
72 {
73         char *leaks = talloc_strdup(lines, "");
74         unsigned int i;
75
76         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
77                 if (strstr(lines[i], " lost ")) {
78                         /* A leak... */
79                         if (strstr(lines[i], " definitely lost ")) {
80                                 /* Definite leak, report. */
81                                 while (lines[i] && !blank_line(lines[i])) {
82                                         leaks = talloc_append_string(leaks,
83                                                                      lines[i]);
84                                         leaks = talloc_append_string(leaks,
85                                                                      "\n");
86                                         i++;
87                                 }
88                         } else
89                                 /* Not definite, ignore. */
90                                 while (lines[i] && !blank_line(lines[i]))
91                                         i++;
92                 } else {
93                         /* A real error. */
94                         while (lines[i] && !blank_line(lines[i])) {
95                                 *errs = talloc_append_string(*errs, lines[i]);
96                                 *errs = talloc_append_string(*errs, "\n");
97                                 i++;
98                         }
99                 }
100         }
101         return leaks;
102 }
103
104 /* Returns leaks, and sets errs[] */
105 static char *analyze_output(const char *output, char **errs)
106 {
107         char *leaks = talloc_strdup(output, "");
108         unsigned int i;
109         char **lines = strsplit(output, output, "\n");
110
111         *errs = talloc_strdup(output, "");
112         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
113                 unsigned int preflen = strspn(lines[i], "=0123456789");
114                 char *prefix, **sublines;
115
116                 /* Ignore erased lines, or weird stuff. */
117                 if (preflen == 0)
118                         continue;
119
120                 prefix = talloc_strndup(output, lines[i], preflen);
121                 sublines = extract_matching(prefix, lines);
122
123                 leaks = talloc_append_string(leaks, get_leaks(sublines, errs));
124         }
125
126         if (!leaks[0]) {
127                 talloc_free(leaks);
128                 leaks = NULL;
129         }
130         if (!(*errs)[0]) {
131                 talloc_free(*errs);
132                 *errs = NULL;
133         }
134         return leaks;
135 }
136
137 static const char *concat(struct score *score, char *bits[])
138 {
139         unsigned int i;
140         char *ret = talloc_strdup(score, "");
141
142         for (i = 0; bits[i]; i++) {
143                 if (i)
144                         ret = talloc_append_string(ret, " ");
145                 ret = talloc_append_string(ret, bits[i]);
146         }
147         return ret;
148 }
149
150 /* FIXME: Run examples, too! */
151 static void do_run_tests_vg(struct manifest *m,
152                              bool keep,
153                             unsigned int *timeleft,
154                             struct score *score)
155 {
156         struct ccan_file *i;
157         struct list_head *list;
158         char *cmdout;
159
160         /* This is slow, so we run once but grab leak info. */
161         score->total = 0;
162         score->pass = true;
163         foreach_ptr(list, &m->run_tests, &m->api_tests) {
164                 list_for_each(list, i, list) {
165                         char *output, *err, *log;
166                         bool pass;
167                         const char *options;
168
169                         score->total++;
170                         options = concat(score,
171                                          per_file_options(&tests_pass_valgrind,
172                                                           i));
173                         if (streq(options, "FAIL"))
174                                 continue;
175
176                         /* FIXME: Valgrind's output sucks.  XML is unreadable by
177                          * humans *and* doesn't support children reporting. */
178                         log = talloc_asprintf(score,
179                                               "%s.valgrind-log", i->compiled);
180                         if (!keep)
181                                 talloc_set_destructor(log,
182                                                       unlink_file_destructor);
183
184                         pass = run_command(score, timeleft, &cmdout,
185                                          "valgrind -q --error-exitcode=101"
186                                            " --leak-check=full"
187                                            " --log-fd=3 %s %s"
188                                            " 3> %s",
189                                            options,
190                                            i->compiled, log);
191                         output = grab_file(i, log, NULL);
192                         /* No valgrind errors?  Expect it to pass... */
193                         if (!output || output[0] == '\0') {
194                                 if (!pass) {
195                                         err = talloc_asprintf(score,
196                                                               "Test failed:\n"
197                                                               "%s",
198                                                               cmdout);
199                                 } else
200                                         err = NULL;
201                                 i->leak_info = NULL;
202                         } else {
203                                 i->leak_info = analyze_output(output, &err);
204                         }
205                         if (err) {
206                                 score_file_error(score, i, 0, "%s", err);
207                                 score->pass = false;
208                         } else
209                                 score->score++;
210                 }
211         }
212 }
213
214 static void do_leakcheck_vg(struct manifest *m,
215                             bool keep,
216                             unsigned int *timeleft,
217                             struct score *score)
218 {
219         struct ccan_file *i;
220         struct list_head *list;
221         char **options;
222         bool leaks = false;
223
224         foreach_ptr(list, &m->run_tests, &m->api_tests) {
225                 list_for_each(list, i, list) {
226                         options = per_file_options(&tests_pass_valgrind_noleaks,
227                                                    i);
228                         if (options[0]) {
229                                 if (streq(options[0], "FAIL")) {
230                                         leaks = true;
231                                         continue;
232                                 }
233                                 errx(1, "Unknown leakcheck options '%s'",
234                                      options[0]);
235                         }
236
237                         if (i->leak_info) {
238                                 score_file_error(score, i, 0, "%s",
239                                                  i->leak_info);
240                                 leaks = true;
241                         }
242                 }
243         }
244
245         /* FIXME: We don't fail for this, since many tests leak. */ 
246         score->pass = true;
247         if (!leaks) {
248                 score->score = 1;
249         }
250 }
251
252 /* Gcc's warn_unused_result is fascist bullshit. */
253 #define doesnt_matter()
254
255 static void run_under_debugger_vg(struct manifest *m, struct score *score)
256 {
257         struct file_error *first;
258         char *command;
259
260         /* Don't ask anything if they suppressed tests. */
261         if (score->pass)
262                 return;
263
264         if (!ask("Should I run the first failing test under the debugger?"))
265                 return;
266
267         first = list_top(&score->per_file_errors, struct file_error, list);
268         command = talloc_asprintf(m, "valgrind --leak-check=full --db-attach=yes%s %s",
269                                   concat(score,
270                                          per_file_options(&tests_pass_valgrind,
271                                                           first->file)),
272                                   first->file->compiled);
273         if (system(command))
274                 doesnt_matter();
275 }
276
277 struct ccanlint tests_pass_valgrind = {
278         .key = "tests_pass_valgrind",
279         .name = "Module's run and api tests succeed under valgrind",
280         .can_run = can_run_vg,
281         .check = do_run_tests_vg,
282         .handle = run_under_debugger_vg,
283         .takes_options = true,
284         .needs = "tests_pass"
285 };
286
287 struct ccanlint tests_pass_valgrind_noleaks = {
288         .key = "tests_pass_valgrind_noleaks",
289         .name = "Module's run and api tests have no memory leaks",
290         .check = do_leakcheck_vg,
291         .takes_options = true,
292         .needs = "tests_pass_valgrind"
293 };
294