ccanlint: try running example code.
[ccan] / tools / ccanlint / tests / examples_run.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/foreach/foreach.h>
5 #include <ccan/str/str.h>
6 #include <sys/types.h>
7 #include <sys/stat.h>
8 #include <fcntl.h>
9 #include <stdint.h>
10 #include <string.h>
11 #include <unistd.h>
12 #include <ctype.h>
13 #include <assert.h>
14
15 static const char *can_run(struct manifest *m)
16 {
17         if (safe_mode)
18                 return "Safe mode enabled";
19         return NULL;
20 }
21
22 struct score {
23         unsigned int score;
24         char *errors;
25 };
26
27 /* Very dumb scanner, allocates %s-strings. */
28 static bool scan_forv(const void *ctx,
29                       const char *input, const char *fmt, const va_list *args)
30 {
31         va_list ap;
32         bool ret;
33
34         if (input[0] == '\0' || fmt[0] == '\0')
35                 return input[0] == fmt[0];
36
37         va_copy(ap, *args);
38
39         if (isspace(fmt[0])) {
40                 /* One format space can swallow many input spaces */
41                 ret = false;
42                 while (isspace(input[0])) {
43                         if (scan_forv(ctx, ++input, fmt+1, &ap)) {
44                                 ret = true;
45                                 break;
46                         }
47                 }
48         } else if (fmt[0] != '%') {
49                 if (toupper(input[0]) != toupper(fmt[0]))
50                         ret = false;
51                 else
52                         ret = scan_forv(ctx, input+1, fmt+1, &ap);
53         } else {
54                 char **p = va_arg(ap, char **);
55                 unsigned int len;
56
57                 ret = false;
58                 assert(fmt[1] == 's');
59                 for (len = 1; input[len-1]; len++) {
60                         ret = scan_forv(ctx, input + len, fmt+2, &ap);
61                         if (ret) {
62                                 *p = talloc_strndup(ctx, input, len);
63                                 ret = true;
64                                 break;
65                         }
66                 }
67         }
68         va_end(ap);
69         return ret;
70 }
71
72 static bool scan_for(const void *ctx, const char *input, const char *fmt, ...)
73 {
74         bool ret;
75         va_list ap;
76
77         va_start(ap, fmt);
78         ret = scan_forv(ctx, input, fmt, &ap);
79         va_end(ap);
80         return ret;
81 }
82
83 static char *find_expect(struct ccan_file *file,
84                          char **lines, char **input, bool *exact,
85                          unsigned *line)
86 {
87         char *expect;
88         const char *fmt;
89
90         for (; lines[*line]; (*line)++) {
91                 char *p = lines[*line] + strspn(lines[*line], " \t");
92                 if (!strstarts(p, "//"))
93                         continue;
94                 p += strspn(p, "/ ");
95                 foreach_ptr(fmt,
96                             "given '%s', outputs '%s'",
97                             "given '%s' outputs '%s'",
98                             "given \"%s\", outputs \"%s\"",
99                             "given \"%s\" outputs \"%s\"") {
100                         if (scan_for(file, p, fmt, input, &expect)) {
101                                 *exact = true;
102                                 return expect;
103                         }
104                 }
105
106                 foreach_ptr(fmt,
107                             "given '%s', output contains '%s'",
108                             "given '%s' output contains '%s'",
109                             "given \"%s\", output contains \"%s\"",
110                             "given \"%s\" output contains \"%s\"") {
111                         if (scan_for(file, p, fmt, input, &expect)) {
112                                 *exact = false;
113                                 return expect;
114                         }
115                 }
116
117                 foreach_ptr(fmt, "outputs '%s'", "outputs \"%s\"") {
118                         if (scan_for(file, p, fmt, &expect)) {
119                                 *input = "";
120                                 *exact = true;
121                                 return expect;
122                         }
123                 }
124
125                 foreach_ptr(fmt,
126                             "given '%s', output contains '%s'",
127                             "given '%s' output contains '%s'",
128                             "given \"%s\", output contains \"%s\"",
129                             "given \"%s\" output contains \"%s\"") {
130                         if (scan_for(file, p, fmt, input, &expect)) {
131                                 *exact = false;
132                                 return expect;
133                         }
134                 }
135
136                 /* Unquoted versions... we can get this wrong! */
137                 foreach_ptr(fmt,
138                             "given %s, outputs '%s'",
139                             "given '%s', outputs %s",
140                             "given %s, outputs \"%s\"",
141                             "given \"%s\", outputs %s",
142                             "given %s, outputs %s",
143                             "given %s outputs '%s'",
144                             "given '%s' outputs %s",
145                             "given %s outputs \"%s\"",
146                             "given \"%s\" outputs %s",
147                             "given %s outputs %s") {
148                         if (scan_for(file, p, fmt, input, &expect)) {
149                                 *exact = true;
150                                 return expect;
151                         }
152                 }
153
154                 foreach_ptr(fmt,
155                             "given %s, output contains '%s'",
156                             "given '%s', output contains %s",
157                             "given %s, output contains \"%s\"",
158                             "given \"%s\", output contains %s",
159                             "given %s, output contains %s",
160                             "given %s output contains '%s'",
161                             "given '%s' output contains %s",
162                             "given %s output contains \"%s\"",
163                             "given \"%s\" output contains %s",
164                             "given %s output contains %s") {
165                         if (scan_for(file, p, fmt, input, &expect)) {
166                                 *exact = false;
167                                 return expect;
168                         }
169                 }
170
171                 foreach_ptr(fmt,
172                             "outputs '%s'",
173                             "outputs \"%s\"",
174                             "outputs %s") {
175                         if (scan_for(file, p, fmt, &expect)) {
176                                 *input = "";
177                                 *exact = true;
178                                 return expect;
179                         }
180                 }
181
182                 foreach_ptr(fmt,
183                             "output contains '%s'",
184                             "output contains \"%s\"",
185                             "output contains %s") {
186                         if (scan_for(file, p, fmt, &expect)) {
187                                 *input = "";
188                                 *exact = false;
189                                 return expect;
190                         }
191                 }
192         }               
193         return NULL;
194 }
195
196 static char *trim(char *string)
197 {
198         while (strends(string, "\n"))
199                string[strlen(string)-1] = '\0';
200         return string;
201 }
202
203 static char *unexpected(struct ccan_file *i, const char *input,
204                         const char *expect, bool exact)
205 {
206         char *output, *cmd;
207         bool ok;
208         unsigned int default_time = default_timeout_ms;
209
210         cmd = talloc_asprintf(i, "echo '%s' | %s %s",
211                               input, i->compiled, input);
212
213         output = run_with_timeout(i, cmd, &ok, &default_time);
214         if (!ok)
215                 return talloc_asprintf(i, "Exited with non-zero status\n");
216
217         if (exact) {
218                 if (streq(output, expect) || streq(trim(output), expect))
219                         return NULL;
220         } else {
221                 if (strstr(output, expect))
222                         return NULL;
223         }
224         return output;
225 }
226
227 static void *run_examples(struct manifest *m, bool keep,
228                           unsigned int *timeleft)
229 {
230         struct ccan_file *i;
231         struct list_head *list;
232         struct score *score = talloc(m, struct score);
233
234         score->score = 0;
235         score->errors = talloc_strdup(score, "");
236
237         examples_run.total_score = 0;
238         foreach_ptr(list, &m->examples, &m->mangled_examples) {
239                 list_for_each(list, i, list) {
240                         char **lines, *expect, *input, *output;
241                         unsigned int linenum = 0;
242                         bool exact;
243
244                         if (i->compiled == NULL)
245                                 continue;
246
247                         lines = get_ccan_file_lines(i);
248
249                         for (expect = find_expect(i, lines, &input, &exact,
250                                                   &linenum);
251                              expect;
252                              linenum++,
253                                      expect = find_expect(i, lines, &input,
254                                                           &exact, &linenum)) {
255                                 examples_run.total_score++;
256                                 output = unexpected(i, input, expect, exact);
257                                 if (!output)
258                                         score->score++;
259                                 else {
260                                         score->errors = talloc_asprintf_append(
261                                                 score->errors,
262                                                 "%s: output '%s' didn't"
263                                                 " %s '%s'\n",
264                                                 i->name, output,
265                                                 exact ? "match" : "contain",
266                                                 expect);
267                                 }
268                         }
269                 }
270         }
271
272         if (strcmp(score->errors, "") == 0) {
273                 talloc_free(score);
274                 return NULL;
275         }
276         return score;
277 }
278
279 static unsigned int score_examples(struct manifest *m, void *check_result)
280 {
281         struct score *score = check_result;
282         return score->score;
283 }
284
285 static const char *describe(struct manifest *m, void *check_result)
286 {
287         struct score *score = check_result;
288         if (verbose)
289                 return talloc_asprintf(m, "Wrong output running examples:\n"
290                                        "%s", score->errors);
291         return NULL;
292 }
293
294 struct ccanlint examples_run = {
295         .key = "examples-run",
296         .name = "Module examples with expected output give that output",
297         .score = score_examples,
298         .total_score = 3, /* This gets changed to # testable, if we run. */
299         .check = run_examples,
300         .describe = describe,
301         .can_run = can_run,
302 };
303
304 REGISTER_TEST(examples_run, &examples_compile, NULL);