d248aaa85fea4c86782057b99ef286c7bf436bc9
[ccan] / tools / ccanlint / tests / tests_pass_valgrind.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str/str.h>
5 #include <ccan/foreach/foreach.h>
6 #include <ccan/grab_file/grab_file.h>
7 #include <ccan/str_talloc/str_talloc.h>
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <fcntl.h>
11 #include <unistd.h>
12 #include <limits.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <err.h>
17 #include <string.h>
18 #include <ctype.h>
19
20 REGISTER_TEST(tests_pass_valgrind);
21
22 /* Note: we already test safe_mode in run_tests.c */
23 static const char *can_run_vg(struct manifest *m)
24 {
25         unsigned int timeleft = default_timeout_ms;
26         char *output;
27
28         if (!run_command(m, &timeleft, &output,
29                          "valgrind -q --error-exitcode=0 true"))
30                 return talloc_asprintf(m, "No valgrind support: %s", output);
31         return NULL;
32 }
33
34 /* Example output:
35 ==2749== Conditional jump or move depends on uninitialised value(s)
36 ==2749==    at 0x4026C60: strnlen (mc_replace_strmem.c:263)
37 ==2749==    by 0x40850E3: vfprintf (vfprintf.c:1614)
38 ==2749==    by 0x408EACF: printf (printf.c:35)
39 ==2749==    by 0x8048465: main (in /tmp/foo)
40 ==2749== 
41 ==2749== 1 bytes in 1 blocks are definitely lost in loss record 1 of 1
42 ==2749==    at 0x4025BD3: malloc (vg_replace_malloc.c:236)
43 ==2749==    by 0x8048444: main (in /tmp/foo)
44 ==2749== 
45 */
46
47 static bool blank_line(const char *line)
48 {
49         return line[strspn(line, "=0123456789 ")] == '\0';
50 }
51
52 /* Removes matching lines from lines array, returns them.  FIXME: Hacky. */
53 static char **extract_matching(const char *prefix, char *lines[])
54 {
55         unsigned int i, num_ret = 0;
56         char **ret = talloc_array(lines, char *, talloc_array_length(lines));
57
58         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
59                 if (strstarts(lines[i], prefix)) {
60                         ret[num_ret++] = talloc_steal(ret, lines[i]);
61                         lines[i] = (char *)"";
62                 }
63         }
64         ret[num_ret++] = NULL;
65
66         /* Make sure length is correct! */
67         return talloc_realloc(NULL, ret, char *, num_ret);
68 }
69
70 static char *get_leaks(char *lines[], char **errs)
71 {
72         char *leaks = talloc_strdup(lines, "");
73         unsigned int i;
74
75         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
76                 if (strstr(lines[i], " lost ")) {
77                         /* A leak... */
78                         if (strstr(lines[i], " definitely lost ")) {
79                                 /* Definite leak, report. */
80                                 while (lines[i] && !blank_line(lines[i])) {
81                                         leaks = talloc_append_string(leaks,
82                                                                      lines[i]);
83                                         leaks = talloc_append_string(leaks,
84                                                                      "\n");
85                                         i++;
86                                 }
87                         } else
88                                 /* Not definite, ignore. */
89                                 while (lines[i] && !blank_line(lines[i]))
90                                         i++;
91                 } else {
92                         /* A real error. */
93                         while (lines[i] && !blank_line(lines[i])) {
94                                 *errs = talloc_append_string(*errs, lines[i]);
95                                 *errs = talloc_append_string(*errs, "\n");
96                                 i++;
97                         }
98                 }
99         }
100         return leaks;
101 }
102
103 /* Returns leaks, and sets errs[] */
104 static char *analyze_output(const char *output, char **errs)
105 {
106         char *leaks = talloc_strdup(output, "");
107         unsigned int i;
108         char **lines = strsplit(output, output, "\n");
109
110         *errs = talloc_strdup(output, "");
111         for (i = 0; i < talloc_array_length(lines) - 1; i++) {
112                 unsigned int preflen = strspn(lines[i], "=0123456789");
113                 char *prefix, **sublines;
114
115                 /* Ignore erased lines, or weird stuff. */
116                 if (preflen == 0)
117                         continue;
118
119                 prefix = talloc_strndup(output, lines[i], preflen);
120                 sublines = extract_matching(prefix, lines);
121
122                 leaks = talloc_append_string(leaks, get_leaks(sublines, errs));
123         }
124
125         if (!leaks[0]) {
126                 talloc_free(leaks);
127                 leaks = NULL;
128         }
129         if (!(*errs)[0]) {
130                 talloc_free(*errs);
131                 *errs = NULL;
132         }
133         return leaks;
134 }
135
136 static const char *concat(struct score *score, char *bits[])
137 {
138         unsigned int i;
139         char *ret = talloc_strdup(score, "");
140
141         for (i = 0; bits[i]; i++) {
142                 if (i)
143                         ret = talloc_append_string(ret, " ");
144                 ret = talloc_append_string(ret, bits[i]);
145         }
146         return ret;
147 }
148
149 /* FIXME: Run examples, too! */
150 static void do_run_tests_vg(struct manifest *m,
151                              bool keep,
152                             unsigned int *timeleft,
153                             struct score *score)
154 {
155         struct ccan_file *i;
156         struct list_head *list;
157         char *cmdout;
158
159         /* This is slow, so we run once but grab leak info. */
160         score->total = 0;
161         score->pass = true;
162         foreach_ptr(list, &m->run_tests, &m->api_tests) {
163                 list_for_each(list, i, list) {
164                         char *output, *err, *log;
165                         bool pass;
166                         const char *options;
167
168                         score->total++;
169                         options = concat(score,
170                                          per_file_options(&tests_pass_valgrind,
171                                                           i));
172                         if (streq(options, "FAIL"))
173                                 continue;
174
175                         /* FIXME: Valgrind's output sucks.  XML is unreadable by
176                          * humans *and* doesn't support children reporting. */
177                         log = talloc_asprintf(score,
178                                               "%s.valgrind-log", i->compiled);
179                         if (!keep)
180                                 talloc_set_destructor(log,
181                                                       unlink_file_destructor);
182
183                         pass = run_command(score, timeleft, &cmdout,
184                                          "valgrind -q --error-exitcode=101"
185                                            " --leak-check=full"
186                                            " --log-fd=3 %s %s"
187                                            " 3> %s",
188                                            options,
189                                            i->compiled, log);
190                         output = grab_file(i, log, NULL);
191                         /* No valgrind errors?  Expect it to pass... */
192                         if (!output || output[0] == '\0') {
193                                 if (!pass) {
194                                         err = talloc_asprintf(score,
195                                                               "Test failed:\n"
196                                                               "%s",
197                                                               cmdout);
198                                 } else
199                                         err = NULL;
200                                 i->leak_info = NULL;
201                         } else {
202                                 i->leak_info = analyze_output(output, &err);
203                         }
204                         if (err) {
205                                 score_file_error(score, i, 0, "%s", err);
206                                 score->pass = false;
207                         } else
208                                 score->score++;
209                 }
210         }
211 }
212
213 static void do_leakcheck_vg(struct manifest *m,
214                             bool keep,
215                             unsigned int *timeleft,
216                             struct score *score)
217 {
218         struct ccan_file *i;
219         struct list_head *list;
220         bool leaks = false;
221
222         foreach_ptr(list, &m->run_tests, &m->api_tests) {
223                 list_for_each(list, i, list) {
224                         if (i->leak_info) {
225                                 score_file_error(score, i, 0, "%s",
226                                                  i->leak_info);
227                                 leaks = true;
228                         }
229                 }
230         }
231
232         /* FIXME: We don't fail for this, since many tests leak. */ 
233         score->pass = true;
234         if (!leaks) {
235                 score->score = 1;
236         }
237 }
238
239 /* Gcc's warn_unused_result is fascist bullshit. */
240 #define doesnt_matter()
241
242 static void run_under_debugger_vg(struct manifest *m, struct score *score)
243 {
244         struct file_error *first;
245         char *command;
246
247         /* Don't ask anything if they suppressed tests. */
248         if (score->pass)
249                 return;
250
251         if (!ask("Should I run the first failing test under the debugger?"))
252                 return;
253
254         first = list_top(&score->per_file_errors, struct file_error, list);
255         command = talloc_asprintf(m, "valgrind --leak-check=full --db-attach=yes%s %s",
256                                   concat(score,
257                                          per_file_options(&tests_pass_valgrind,
258                                                           first->file)),
259                                   first->file->compiled);
260         if (system(command))
261                 doesnt_matter();
262 }
263
264 struct ccanlint tests_pass_valgrind = {
265         .key = "tests_pass_valgrind",
266         .name = "Module's run and api tests succeed under valgrind",
267         .can_run = can_run_vg,
268         .check = do_run_tests_vg,
269         .handle = run_under_debugger_vg,
270         .takes_options = true,
271         .needs = "tests_pass"
272 };
273
274 struct ccanlint tests_pass_valgrind_noleaks = {
275         .key = "tests_pass_valgrind_noleaks",
276         .name = "Module's run and api tests have no memory leaks",
277         .check = do_leakcheck_vg,
278         .needs = "tests_pass_valgrind"
279 };
280
281 REGISTER_TEST(tests_pass_valgrind_noleaks);