ccanlint: use gcov to rate test coverage (score out of 5)
[ccan] / tools / ccanlint / tests / run-coverage.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str_talloc/str_talloc.h>
5 #include <ccan/str/str.h>
6 #include <sys/types.h>
7 #include <sys/stat.h>
8 #include <fcntl.h>
9 #include <unistd.h>
10 #include <limits.h>
11 #include <errno.h>
12 #include <stdlib.h>
13 #include <stdio.h>
14 #include <err.h>
15 #include <string.h>
16 #include <ctype.h>
17 #include "build-coverage.h"
18
19 struct coverage_result {
20         float uncovered;
21         const char *what;
22         const char *output;
23 };
24
25 static bool find_source_file(struct manifest *m, const char *filename)
26 {
27         struct ccan_file *i;
28
29         list_for_each(&m->c_files, i, list) {
30                 if (streq(i->fullname, filename))
31                         return true;
32         }
33         list_for_each(&m->h_files, i, list) {
34                 if (streq(i->fullname, filename))
35                         return true;
36         }
37         return false;
38 }
39
40 /* FIXME: Don't know how stable this is.  Read cov files directly? */
41 static void analyze_coverage(struct manifest *m,
42                              struct coverage_result *res, const char *output)
43 {
44         char **lines = strsplit(res, output, "\n", NULL);
45         float covered_lines = 0.0;
46         unsigned int i, total_lines = 0;
47         bool lines_matter = false;
48
49         /* FIXME: We assume GCOV mentions all files!
50           Output looks like:
51            File '../../../ccan/tdb2/private.h'
52            Lines executed:0.00% of 8
53
54            File '../../../ccan/tdb2/tdb.c'
55            Lines executed:0.00% of 450
56         */
57
58         for (i = 0; lines[i]; i++) {
59                 if (strstarts(lines[i], "File '")) {
60                         char *file = lines[i] + strlen("File '");
61                         file[strcspn(file, "'")] = '\0';
62                         if (find_source_file(m, file))
63                                 lines_matter = true;
64                         else
65                                 lines_matter = false;
66                 } else if (lines_matter
67                            && strstarts(lines[i], "Lines executed:")) {
68                         float ex;
69                         unsigned of;
70                         if (sscanf(lines[i], "Lines executed:%f%% of %u",
71                                    &ex, &of) != 2)
72                                 errx(1, "Could not parse line '%s'", lines[i]);
73                         total_lines += of;
74                         covered_lines += ex / 100.0 * of;
75                 }
76         }
77
78         /* Nothing covered? */
79         if (total_lines == 0)
80                 res->uncovered = 1.0;
81         else
82                 res->uncovered = 1.0 - covered_lines / total_lines;
83 }
84
85 static void *do_run_coverage_tests(struct manifest *m,
86                                    bool keep,
87                                    unsigned int *timeleft)
88 {
89         struct coverage_result *res;
90         struct ccan_file *i;
91         char *cmdout;
92         char *olddir;
93         char *covcmd;
94         bool ok;
95
96         /* We run tests in the module directory, so any paths
97          * referenced can all be module-local. */
98         olddir = talloc_getcwd(m);
99         if (!olddir)
100                 err(1, "Could not save cwd");
101         if (chdir(m->dir) != 0)
102                 err(1, "Could not chdir to %s", m->dir);
103
104         res = talloc(m, struct coverage_result);
105         res->what = NULL;
106         res->uncovered = 1.0;
107
108         /* This tells gcov where we put those .gcno files. */
109         covcmd = talloc_asprintf(m, "gcov -n -o %s",
110                                  talloc_dirname(res, m->info_file->compiled));
111
112         /* Run them all. */
113         list_for_each(&m->run_tests, i, list) {
114                 cmdout = run_command(m, timeleft, i->cov_compiled);
115                 if (cmdout) {
116                         res->what = i->fullname;
117                         res->output = talloc_steal(res, cmdout);
118                         return res;
119                 }
120                 covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
121                 move_gcov_turd(olddir, i, ".gcda");
122         }
123
124         list_for_each(&m->api_tests, i, list) {
125                 cmdout = run_command(m, timeleft, i->cov_compiled);
126                 if (cmdout) {
127                         res->what = i->fullname;
128                         res->output = talloc_steal(res, cmdout);
129                         return res;
130                 }
131                 covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
132                 move_gcov_turd(olddir, i, ".gcda");
133         }
134
135         /* Now run gcov: we want output even if it succeeds. */
136         cmdout = run_with_timeout(m, covcmd, &ok, timeleft);
137         if (!ok) {
138                 res->what = "Running gcov";
139                 res->output = talloc_steal(res, cmdout);
140                 return res;
141         }
142
143         analyze_coverage(m, res, cmdout);
144         return res;
145 }
146
147 /* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */
148 static unsigned int score_coverage(struct manifest *m, void *check_result)
149 {
150         struct coverage_result *res = check_result;
151         float thresh;
152         unsigned int i;
153
154         for (i = 0, thresh = 0.5;
155              i < run_coverage_tests.total_score;
156              i++, thresh /= 2) {
157                 if (res->uncovered > thresh)
158                         break;
159         }
160         return i;
161 }
162
163 static const char *describe_run_coverage_tests(struct manifest *m,
164                                                void *check_result)
165 {
166         struct coverage_result *res = check_result;
167
168         if (res->what)
169                 return talloc_asprintf(m, "%s: %s", res->what, res->output);
170
171         return talloc_asprintf(m, "Tests achieved %0.2f%% coverage",
172                                (1.0 - res->uncovered) * 100);
173 }
174
175 struct ccanlint run_coverage_tests = {
176         .key = "test-coverage",
177         .name = "Code coverage of module tests",
178         .total_score = 5,
179         .score = score_coverage,
180         .check = do_run_coverage_tests,
181         .describe = describe_run_coverage_tests,
182 };
183
184 REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);