ccanlint: chdir to temporary dir so gcov files land there.
[ccan] / tools / ccanlint / tests / run-coverage.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str_talloc/str_talloc.h>
5 #include <ccan/str/str.h>
6 #include <sys/types.h>
7 #include <sys/stat.h>
8 #include <fcntl.h>
9 #include <unistd.h>
10 #include <limits.h>
11 #include <errno.h>
12 #include <stdlib.h>
13 #include <stdio.h>
14 #include <err.h>
15 #include <string.h>
16 #include <ctype.h>
17
18 struct coverage_result {
19         float uncovered;
20         const char *what;
21         const char *output;
22 };
23
24 static bool find_source_file(struct manifest *m, const char *filename)
25 {
26         struct ccan_file *i;
27
28         list_for_each(&m->c_files, i, list) {
29                 if (streq(i->fullname, filename))
30                         return true;
31         }
32         list_for_each(&m->h_files, i, list) {
33                 if (streq(i->fullname, filename))
34                         return true;
35         }
36         return false;
37 }
38
39 /* FIXME: Don't know how stable this is.  Read cov files directly? */
40 static void analyze_coverage(struct manifest *m,
41                              struct coverage_result *res, const char *output)
42 {
43         char **lines = strsplit(res, output, "\n", NULL);
44         float covered_lines = 0.0;
45         unsigned int i, total_lines = 0;
46         bool lines_matter = false;
47
48         /* FIXME: We assume GCOV mentions all files!
49           Output looks like:
50            File '../../../ccan/tdb2/private.h'
51            Lines executed:0.00% of 8
52
53            File '../../../ccan/tdb2/tdb.c'
54            Lines executed:0.00% of 450
55         */
56
57         for (i = 0; lines[i]; i++) {
58                 if (strstarts(lines[i], "File '")) {
59                         char *file = lines[i] + strlen("File '");
60                         file[strcspn(file, "'")] = '\0';
61                         if (find_source_file(m, file))
62                                 lines_matter = true;
63                         else
64                                 lines_matter = false;
65                 } else if (lines_matter
66                            && strstarts(lines[i], "Lines executed:")) {
67                         float ex;
68                         unsigned of;
69                         if (sscanf(lines[i], "Lines executed:%f%% of %u",
70                                    &ex, &of) != 2)
71                                 errx(1, "Could not parse line '%s'", lines[i]);
72                         total_lines += of;
73                         covered_lines += ex / 100.0 * of;
74                 }
75         }
76
77         /* Nothing covered? */
78         if (total_lines == 0)
79                 res->uncovered = 1.0;
80         else
81                 res->uncovered = 1.0 - covered_lines / total_lines;
82 }
83
84 static void *do_run_coverage_tests(struct manifest *m,
85                                    bool keep,
86                                    unsigned int *timeleft)
87 {
88         struct coverage_result *res;
89         struct ccan_file *i;
90         char *cmdout;
91         char *olddir;
92         char *covcmd;
93         bool ok;
94
95         /* We run tests in the module directory, so any paths
96          * referenced can all be module-local. */
97         olddir = talloc_getcwd(m);
98         if (!olddir)
99                 err(1, "Could not save cwd");
100         if (chdir(m->dir) != 0)
101                 err(1, "Could not chdir to %s", m->dir);
102
103         res = talloc(m, struct coverage_result);
104         res->what = NULL;
105         res->uncovered = 1.0;
106
107         /* This tells gcov where we put those .gcno files. */
108         covcmd = talloc_asprintf(m, "gcov -n -o %s",
109                                  talloc_dirname(res, m->info_file->compiled));
110
111         /* Run them all. */
112         list_for_each(&m->run_tests, i, list) {
113                 cmdout = run_command(m, timeleft, i->cov_compiled);
114                 if (cmdout) {
115                         res->what = i->fullname;
116                         res->output = talloc_steal(res, cmdout);
117                         return res;
118                 }
119                 covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
120         }
121
122         list_for_each(&m->api_tests, i, list) {
123                 cmdout = run_command(m, timeleft, i->cov_compiled);
124                 if (cmdout) {
125                         res->what = i->fullname;
126                         res->output = talloc_steal(res, cmdout);
127                         return res;
128                 }
129                 covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
130         }
131
132         /* Now run gcov: we want output even if it succeeds. */
133         cmdout = run_with_timeout(m, covcmd, &ok, timeleft);
134         if (!ok) {
135                 res->what = "Running gcov";
136                 res->output = talloc_steal(res, cmdout);
137                 return res;
138         }
139
140         analyze_coverage(m, res, cmdout);
141         return res;
142 }
143
144 /* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */
145 static unsigned int score_coverage(struct manifest *m, void *check_result)
146 {
147         struct coverage_result *res = check_result;
148         float thresh;
149         unsigned int i;
150
151         for (i = 0, thresh = 0.5;
152              i < run_coverage_tests.total_score;
153              i++, thresh /= 2) {
154                 if (res->uncovered > thresh)
155                         break;
156         }
157         return i;
158 }
159
160 static const char *describe_run_coverage_tests(struct manifest *m,
161                                                void *check_result)
162 {
163         struct coverage_result *res = check_result;
164
165         if (res->what)
166                 return talloc_asprintf(m, "%s: %s", res->what, res->output);
167
168         return talloc_asprintf(m, "Tests achieved %0.2f%% coverage",
169                                (1.0 - res->uncovered) * 100);
170 }
171
172 struct ccanlint run_coverage_tests = {
173         .key = "test-coverage",
174         .name = "Code coverage of module tests",
175         .total_score = 5,
176         .score = score_coverage,
177         .check = do_run_coverage_tests,
178         .describe = describe_run_coverage_tests,
179 };
180
181 REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);