json_out: make json_out_finished finish buffer.
[ccan] / tools / ccanlint / tests / tests_pass_valgrind.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/str/str.h>
4 #include <ccan/take/take.h>
5 #include <ccan/foreach/foreach.h>
6 #include <ccan/tal/grab_file/grab_file.h>
7 #include "tests_pass.h"
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <fcntl.h>
11 #include <unistd.h>
12 #include <limits.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <err.h>
17 #include <string.h>
18 #include <ctype.h>
19
20 /* Note: we already test safe_mode in run_tests.c */
21 static const char *can_run_vg(struct manifest *m)
22 {
23         if (!do_valgrind)
24                 return tal_fmt(m, "No valgrind support");
25         return NULL;
26 }
27
28 static void do_leakcheck_vg(struct manifest *m,
29                             unsigned int *timeleft,
30                             struct score *score);
31
32 static struct ccanlint tests_pass_valgrind_noleaks = {
33         .key = "tests_pass_valgrind_noleaks",
34         .name = "Module's run and api tests have no memory leaks",
35         .check = do_leakcheck_vg,
36         .takes_options = true,
37         .needs = "tests_pass_valgrind"
38 };
39 REGISTER_TEST(tests_pass_valgrind_noleaks);
40
41
42 /* Example output:
43 ==2749== Conditional jump or move depends on uninitialised value(s)
44 ==2749==    at 0x4026C60: strnlen (mc_replace_strmem.c:263)
45 ==2749==    by 0x40850E3: vfprintf (vfprintf.c:1614)
46 ==2749==    by 0x408EACF: printf (printf.c:35)
47 ==2749==    by 0x8048465: main (in /tmp/foo)
48 ==2749== 
49 ==2749== 1 bytes in 1 blocks are definitely lost in loss record 1 of 1
50 ==2749==    at 0x4025BD3: malloc (vg_replace_malloc.c:236)
51 ==2749==    by 0x8048444: main (in /tmp/foo)
52 ==2749== 
53 */
54
55 static bool blank_line(const char *line)
56 {
57         return line[strspn(line, "=0123456789 ")] == '\0';
58 }
59
60 /* Removes matching lines from lines array, returns them.  FIXME: Hacky. */
61 static char **extract_matching(const char *prefix, char *lines[])
62 {
63         unsigned int i, num_ret = 0;
64         char **ret = tal_arr(lines, char *, tal_count(lines));
65
66         for (i = 0; i < tal_count(lines) - 1; i++) {
67                 if (strstarts(lines[i], prefix)) {
68                         ret[num_ret++] = tal_strdup(ret, lines[i]);
69                         lines[i] = (char *)"";
70                 }
71         }
72         ret[num_ret++] = NULL;
73
74         /* Make sure tal_count is correct! */
75         tal_resize(&ret, num_ret);
76         return ret;
77 }
78
79 static char *get_leaks(char *lines[], char **errs)
80 {
81         char *leaks = tal_strdup(lines, "");
82         unsigned int i;
83
84         for (i = 0; i < tal_count(lines) - 1; i++) {
85                 if (strstr(lines[i], " lost ")) {
86                         /* A leak... */
87                         if (strstr(lines[i], " definitely lost ")) {
88                                 /* Definite leak, report. */
89                                 while (lines[i] && !blank_line(lines[i])) {
90                                         tal_append_fmt(&leaks, "%s\n",
91                                                        lines[i]);
92                                         i++;
93                                 }
94                         } else
95                                 /* Not definite, ignore. */
96                                 while (lines[i] && !blank_line(lines[i]))
97                                         i++;
98                 } else {
99                         /* A real error. */
100                         while (lines[i] && !blank_line(lines[i])) {
101                                 if (!*errs)
102                                         *errs = tal_fmt(NULL, "%s\n", lines[i]);
103                                 else
104                                         tal_append_fmt(errs, "%s\n", lines[i]);
105                                 i++;
106                         }
107                 }
108         }
109         return leaks;
110 }
111
112 /* Returns leaks, and sets errs[] */
113 static char *analyze_output(const char *output, char **errs)
114 {
115         char *leaks = tal_strdup(output, "");
116         unsigned int i;
117         char **lines = tal_strsplit(output, output, "\n", STR_EMPTY_OK);
118
119         *errs = tal_strdup(output, "");
120         for (i = 0; i < tal_count(lines) - 1; i++) {
121                 unsigned int preflen = strspn(lines[i], "=0123456789");
122                 char *prefix, **sublines;
123
124                 /* Ignore erased lines, or weird stuff. */
125                 if (preflen == 0)
126                         continue;
127
128                 prefix = tal_strndup(output, lines[i], preflen);
129                 sublines = extract_matching(prefix, lines);
130
131                 leaks = tal_strcat(output, take(leaks),
132                                    take(get_leaks(sublines, errs)));
133         }
134
135         if (!leaks[0]) {
136                 tal_free(leaks);
137                 leaks = NULL;
138         }
139         if (!(*errs)[0]) {
140                 tal_free(*errs);
141                 *errs = NULL;
142         }
143         return leaks;
144 }
145
146 static const char *concat(struct score *score, char *bits[])
147 {
148         unsigned int i;
149         char *ret = tal_strdup(score, "");
150
151         for (i = 0; bits[i]; i++) {
152                 if (i)
153                         ret = tal_strcat(score, take(ret), " ");
154                 ret = tal_strcat(score, take(ret), bits[i]);
155         }
156         return ret;
157 }
158
159 /* FIXME: Run examples, too! */
160 static void do_run_tests_vg(struct manifest *m,
161                             unsigned int *timeleft UNNEEDED,
162                             struct score *score)
163 {
164         struct ccan_file *i;
165         struct list_head *list;
166
167         /* This is slow, so we run once but grab leak info. */
168         score->total = 0;
169         score->pass = true;
170         foreach_ptr(list, &m->run_tests, &m->api_tests) {
171                 list_for_each(list, i, list) {
172                         char *err, *output;
173                         const char *options;
174
175                         score->total++;
176                         options = concat(score,
177                                          per_file_options(&tests_pass_valgrind,
178                                                           i));
179                         if (streq(options, "FAIL")) {
180                                 i->leak_info = NULL;
181                                 continue;
182                         }
183
184                         output = grab_file(i, i->valgrind_log);
185                         /* No valgrind errors? */
186                         if (!output || output[0] == '\0') {
187                                 err = NULL;
188                                 i->leak_info = NULL;
189                         } else {
190                                 i->leak_info = analyze_output(output, &err);
191                         }
192                         if (err) {
193                                 score_file_error(score, i, 0, "%s", err);
194                                 score->pass = false;
195                         } else
196                                 score->score++;
197                 }
198         }
199 }
200
201 static void do_leakcheck_vg(struct manifest *m,
202                             unsigned int *timeleft UNNEEDED,
203                             struct score *score)
204 {
205         struct ccan_file *i;
206         struct list_head *list;
207         char **options;
208         bool leaks = false;
209
210         foreach_ptr(list, &m->run_tests, &m->api_tests) {
211                 list_for_each(list, i, list) {
212                         options = per_file_options(&tests_pass_valgrind_noleaks,
213                                                    i);
214                         if (options[0]) {
215                                 if (streq(options[0], "FAIL")) {
216                                         leaks = true;
217                                         continue;
218                                 }
219                                 errx(1, "Unknown leakcheck options '%s'",
220                                      options[0]);
221                         }
222
223                         if (i->leak_info) {
224                                 score_file_error(score, i, 0, "%s",
225                                                  i->leak_info);
226                                 leaks = true;
227                         }
228                 }
229         }
230
231         /* FIXME: We don't fail for this, since many tests leak. */ 
232         score->pass = true;
233         if (!leaks) {
234                 score->score = 1;
235         }
236 }
237
238 /* Gcc's warn_unused_result is fascist bullshit. */
239 #define doesnt_matter()
240
241 static void run_under_debugger_vg(struct manifest *m, struct score *score)
242 {
243         struct file_error *first;
244         char *command;
245
246         /* Don't ask anything if they suppressed tests. */
247         if (score->pass)
248                 return;
249
250         if (!ask("Should I run the first failing test under the debugger?"))
251                 return;
252
253         first = list_top(&score->per_file_errors, struct file_error, list);
254         command = tal_fmt(m, "valgrind --leak-check=full --db-attach=yes%s %s %s",
255                           concat(score, per_file_options(&tests_pass_valgrind,
256                                                          first->file)),
257                           valgrind_suppress, first->file->compiled[COMPILE_NORMAL]);
258         if (system(command)) {
259                 doesnt_matter();
260         }
261 }
262
263 struct ccanlint tests_pass_valgrind = {
264         .key = "tests_pass_valgrind",
265         .name = "Module's run and api tests succeed under valgrind",
266         .can_run = can_run_vg,
267         .check = do_run_tests_vg,
268         .handle = run_under_debugger_vg,
269         .takes_options = true,
270         .needs = "tests_pass"
271 };
272 REGISTER_TEST(tests_pass_valgrind);