ccanlint: use familiar names for temporary files, show them with -vv.
[ccan] / tools / ccanlint / tests / run-coverage.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str_talloc/str_talloc.h>
5 #include <ccan/grab_file/grab_file.h>
6 #include <ccan/str/str.h>
7 #include <sys/types.h>
8 #include <sys/stat.h>
9 #include <fcntl.h>
10 #include <unistd.h>
11 #include <limits.h>
12 #include <errno.h>
13 #include <stdlib.h>
14 #include <stdio.h>
15 #include <err.h>
16 #include <string.h>
17 #include <ctype.h>
18
19 struct coverage_result {
20         float uncovered;
21         const char *what;
22         char *output;
23 };
24
25 static bool find_source_file(struct manifest *m, const char *filename)
26 {
27         struct ccan_file *i;
28
29         list_for_each(&m->c_files, i, list) {
30                 if (streq(i->fullname, filename))
31                         return true;
32         }
33         list_for_each(&m->h_files, i, list) {
34                 if (streq(i->fullname, filename))
35                         return true;
36         }
37         return false;
38 }
39
40 /* FIXME: Don't know how stable this is.  Read cov files directly? */
41 static void analyze_coverage(struct manifest *m,
42                              struct coverage_result *res, const char *output,
43                              bool full_gcov)
44 {
45         char **lines = strsplit(res, output, "\n", NULL);
46         float covered_lines = 0.0;
47         unsigned int i, total_lines = 0;
48         bool lines_matter = false;
49
50         /* FIXME: We assume GCOV mentions all files!
51           Output looks like:
52            File '../../../ccan/tdb2/private.h'
53            Lines executed:0.00% of 8
54            /home/ccan/ccan/tdb2/test/run-simple-delete.c:creating 'run-simple-delete.c.gcov'
55
56            File '../../../ccan/tdb2/tdb.c'
57            Lines executed:0.00% of 450
58         */
59
60         for (i = 0; lines[i]; i++) {
61                 if (strstarts(lines[i], "File '")) {
62                         char *file = lines[i] + strlen("File '");
63                         file[strcspn(file, "'")] = '\0';
64                         if (find_source_file(m, file))
65                                 lines_matter = true;
66                         else
67                                 lines_matter = false;
68                 } else if (lines_matter
69                            && strstarts(lines[i], "Lines executed:")) {
70                         float ex;
71                         unsigned of;
72                         if (sscanf(lines[i], "Lines executed:%f%% of %u",
73                                    &ex, &of) != 2)
74                                 errx(1, "Could not parse line '%s'", lines[i]);
75                         total_lines += of;
76                         covered_lines += ex / 100.0 * of;
77                 } else if (full_gcov && strstr(lines[i], ":creating '")) {
78                         char *file, *filename, *apostrophe;
79                         apostrophe = strchr(lines[i], '\'');
80                         filename = apostrophe + 1;
81                         apostrophe = strchr(filename, '\'');
82                         *apostrophe = '\0';
83                         if (lines_matter) {
84                                 file = grab_file(res, filename, NULL);
85                                 if (!file) {
86                                         res->what = talloc_asprintf(res,
87                                                             "Reading %s",
88                                                             filename);
89                                         res->output = talloc_strdup(res,
90                                                             strerror(errno));
91                                         return;
92                                 }
93                                 res->output = talloc_append_string(res->output,
94                                                                    file);
95                         }
96                         if (tools_verbose)
97                                 printf("Unlinking %s", filename);
98                         unlink(filename);
99                 }
100         }
101
102         /* Nothing covered? */
103         if (total_lines == 0)
104                 res->uncovered = 1.0;
105         else
106                 res->uncovered = 1.0 - covered_lines / total_lines;
107 }
108
109 static void *do_run_coverage_tests(struct manifest *m,
110                                    bool keep,
111                                    unsigned int *timeleft)
112 {
113         struct coverage_result *res;
114         struct ccan_file *i;
115         char *cmdout;
116         char *covcmd;
117         bool ok;
118         bool full_gcov = (verbose > 1);
119
120         res = talloc(m, struct coverage_result);
121         res->what = NULL;
122         res->output = talloc_strdup(res, "");
123         res->uncovered = 1.0;
124
125         /* This tells gcov where we put those .gcno files. */
126         covcmd = talloc_asprintf(m, "gcov %s -o %s",
127                                  full_gcov ? "" : "-n",
128                                  talloc_dirname(res, m->info_file->compiled));
129
130         /* Run them all. */
131         list_for_each(&m->run_tests, i, list) {
132                 cmdout = run_command(m, timeleft, i->cov_compiled);
133                 if (cmdout) {
134                         res->what = i->fullname;
135                         res->output = talloc_steal(res, cmdout);
136                         return res;
137                 }
138                 covcmd = talloc_asprintf_append(covcmd, " %s", i->fullname);
139         }
140
141         list_for_each(&m->api_tests, i, list) {
142                 cmdout = run_command(m, timeleft, i->cov_compiled);
143                 if (cmdout) {
144                         res->what = i->fullname;
145                         res->output = talloc_steal(res, cmdout);
146                         return res;
147                 }
148                 covcmd = talloc_asprintf_append(covcmd, " %s", i->fullname);
149         }
150
151         /* Now run gcov: we want output even if it succeeds. */
152         cmdout = run_with_timeout(m, covcmd, &ok, timeleft);
153         if (!ok) {
154                 res->what = "Running gcov";
155                 res->output = talloc_steal(res, cmdout);
156                 return res;
157         }
158
159         analyze_coverage(m, res, cmdout, full_gcov);
160
161         return res;
162 }
163
164 /* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */
165 static unsigned int score_coverage(struct manifest *m, void *check_result)
166 {
167         struct coverage_result *res = check_result;
168         float thresh;
169         unsigned int i;
170
171         for (i = 0, thresh = 0.5;
172              i < run_coverage_tests.total_score;
173              i++, thresh /= 2) {
174                 if (res->uncovered > thresh)
175                         break;
176         }
177         return i;
178 }
179
180 static const char *describe_run_coverage_tests(struct manifest *m,
181                                                void *check_result)
182 {
183         struct coverage_result *res = check_result;
184         bool full_gcov = (verbose > 1);
185         char *ret;
186
187         if (res->what)
188                 return talloc_asprintf(m, "%s: %s", res->what, res->output);
189
190         if (!verbose)
191                 return NULL;
192
193         ret = talloc_asprintf(m, "Tests achieved %0.2f%% coverage",
194                               (1.0 - res->uncovered) * 100);
195         if (full_gcov)
196                 ret = talloc_asprintf_append(ret, "\n%s", res->output);
197         return ret;
198 }
199
200 struct ccanlint run_coverage_tests = {
201         .key = "test-coverage",
202         .name = "Code coverage of module tests",
203         .total_score = 5,
204         .score = score_coverage,
205         .check = do_run_coverage_tests,
206         .describe = describe_run_coverage_tests,
207 };
208
209 REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);