67829e96918f8fe4e1509ac8830fd27a83e2b9cb
[ccan] / tools / ccanlint / tests / tests_coverage.c
1 #include <tools/ccanlint/ccanlint.h>
2 #include <tools/tools.h>
3 #include <ccan/talloc/talloc.h>
4 #include <ccan/str_talloc/str_talloc.h>
5 #include <ccan/grab_file/grab_file.h>
6 #include <ccan/str/str.h>
7 #include <ccan/foreach/foreach.h>
8 #include <sys/types.h>
9 #include <sys/stat.h>
10 #include <fcntl.h>
11 #include <unistd.h>
12 #include <limits.h>
13 #include <errno.h>
14 #include <stdlib.h>
15 #include <stdio.h>
16 #include <err.h>
17 #include <string.h>
18 #include <ctype.h>
19
20 static bool find_source_file(const struct manifest *m, const char *filename)
21 {
22         const struct ccan_file *i;
23
24         list_for_each(&m->c_files, i, list) {
25                 if (streq(i->fullname, filename))
26                         return true;
27         }
28         list_for_each(&m->h_files, i, list) {
29                 if (streq(i->fullname, filename))
30                         return true;
31         }
32         return false;
33 }
34
35 /* 1 point for 50%, 2 points for 75%, 3 points for 87.5%...  Bonus for 100%. */
36 static unsigned int score_coverage(float covered, unsigned total)
37 {
38         float thresh, uncovered = 1.0 - covered;
39         unsigned int i;
40
41         if (covered == 1.0)
42                 return total;
43
44         total--;
45         for (i = 0, thresh = 0.5; i < total; i++, thresh /= 2) {
46                 if (uncovered > thresh)
47                         break;
48         }
49         return i;
50 }
51
52
53 /* FIXME: Don't know how stable this is.  Read cov files directly? */
54 static void analyze_coverage(struct manifest *m, bool full_gcov,
55                              const char *output, struct score *score)
56 {
57         char **lines = strsplit(score, output, "\n");
58         float covered_lines = 0.0;
59         unsigned int i, total_lines = 0;
60         bool lines_matter = false;
61
62         /*
63           Output looks like:
64            File '../../../ccan/tdb2/private.h'
65            Lines executed:0.00% of 8
66            /home/ccan/ccan/tdb2/test/run-simple-delete.c:creating 'run-simple-delete.c.gcov'
67
68            File '../../../ccan/tdb2/tdb.c'
69            Lines executed:0.00% of 450
70         */
71
72         for (i = 0; lines[i]; i++) {
73                 if (strstarts(lines[i], "File '")) {
74                         char *file = lines[i] + strlen("File '");
75                         file[strcspn(file, "'")] = '\0';
76                         if (find_source_file(m, file))
77                                 lines_matter = true;
78                         else
79                                 lines_matter = false;
80                 } else if (lines_matter
81                            && strstarts(lines[i], "Lines executed:")) {
82                         float ex;
83                         unsigned of;
84                         if (sscanf(lines[i], "Lines executed:%f%% of %u",
85                                    &ex, &of) != 2)
86                                 errx(1, "Could not parse line '%s'", lines[i]);
87                         total_lines += of;
88                         covered_lines += ex / 100.0 * of;
89                 } else if (full_gcov && strstr(lines[i], ":creating '")) {
90                         char *file, *filename, *apostrophe;
91                         apostrophe = strchr(lines[i], '\'');
92                         filename = apostrophe + 1;
93                         apostrophe = strchr(filename, '\'');
94                         *apostrophe = '\0';
95                         if (lines_matter) {
96                                 file = grab_file(score, filename, NULL);
97                                 if (!file) {
98                                         score->error = talloc_asprintf(score,
99                                                             "Reading %s",
100                                                             filename);
101                                         return;
102                                 }
103                                 printf("%s", file);
104                         }
105                         if (tools_verbose)
106                                 printf("Unlinking %s", filename);
107                         unlink(filename);
108                 }
109         }
110
111         score->pass = true;
112
113         if (verbose > 1)
114                 printf("%u of %u lines covered\n",
115                        (unsigned)covered_lines, total_lines);
116
117         /* Nothing covered?  We can't tell if there's a source file which
118          * was never executed, or there really is no code to execute, so
119          * assume the latter: this test deserves no score. */
120         if (total_lines == 0)
121                 score->total = score->score = 0;
122         else {
123                 score->total = 6;
124                 score->score = score_coverage(covered_lines / total_lines,
125                                               score->total);
126         }
127 }
128
129 static void do_run_coverage_tests(struct manifest *m,
130                                   bool keep,
131                                   unsigned int *timeleft, struct score *score)
132 {
133         struct ccan_file *i;
134         char *cmdout, *outdir;
135         char *covcmd;
136         bool full_gcov = (verbose > 1);
137         struct list_head *list;
138
139         /* This tells gcov where we put those .gcno files. */
140         outdir = talloc_dirname(score, m->info_file->compiled);
141         covcmd = talloc_asprintf(m, "gcov %s -o %s",
142                                  full_gcov ? "" : "-n",
143                                  outdir);
144
145         /* Unlink these files afterwards. */
146         if (!keep) {
147                 talloc_set_destructor(talloc_asprintf(score,
148                                                       "%s/run.gcno", outdir),
149                                       unlink_file_destructor);
150                 talloc_set_destructor(talloc_asprintf(score,
151                                                       "%s/run.gcda", outdir),
152                                       unlink_file_destructor);
153         }
154
155         /* Run them all. */
156         foreach_ptr(list, &m->run_tests, &m->api_tests) {
157                 list_for_each(list, i, list) {
158                         if (run_command(score, timeleft, &cmdout,
159                                         "%s", i->cov_compiled)) {
160                                 covcmd = talloc_asprintf_append(covcmd, " %s",
161                                                                 i->fullname);
162                         } else {
163                                 score_file_error(score, i, 0,
164                                                  "Running test with coverage"
165                                                  " failed: %s", cmdout);
166                                 return;
167                         }
168                 }
169         }
170
171         /* Now run gcov: we want output even if it succeeds. */
172         if (!run_command(score, timeleft, &cmdout, "%s", covcmd)) {
173                 score->error = talloc_asprintf(score, "Running gcov: %s",
174                                                cmdout);
175                 return;
176         }
177
178         analyze_coverage(m, full_gcov, cmdout, score);
179 }
180
181 struct ccanlint tests_coverage = {
182         .key = "tests_coverage",
183         .name = "Module's tests cover all the code",
184         .check = do_run_coverage_tests,
185         .needs = "tests_compile_coverage tests_pass"
186 };
187
188 REGISTER_TEST(tests_coverage);