git.ozlabs.org Git - ccan/blob - tools/ccanlint/tests/run-coverage.c

   1 #include <tools/ccanlint/ccanlint.h>
   2 #include <tools/tools.h>
   3 #include <ccan/talloc/talloc.h>
   4 #include <ccan/str_talloc/str_talloc.h>
   5 #include <ccan/grab_file/grab_file.h>
   6 #include <ccan/str/str.h>
   7 #include <sys/types.h>
   8 #include <sys/stat.h>
   9 #include <fcntl.h>
  10 #include <unistd.h>
  11 #include <limits.h>
  12 #include <errno.h>
  13 #include <stdlib.h>
  14 #include <stdio.h>
  15 #include <err.h>
  16 #include <string.h>
  17 #include <ctype.h>
  18
  19 struct coverage_result {
  20         float uncovered;
  21         const char *what;
  22         char *output;
  23 };
  24
  25 static bool find_source_file(struct manifest *m, const char *filename)
  26 {
  27         struct ccan_file *i;
  28
  29         list_for_each(&m->c_files, i, list) {
  30                 if (streq(i->fullname, filename))
  31                         return true;
  32         }
  33         list_for_each(&m->h_files, i, list) {
  34                 if (streq(i->fullname, filename))
  35                         return true;
  36         }
  37         return false;
  38 }
  39
  40 /* FIXME: Don't know how stable this is.  Read cov files directly? */
  41 static void analyze_coverage(struct manifest *m,
  42                              struct coverage_result *res, const char *output,
  43                              bool full_gcov)
  44 {
  45         char **lines = strsplit(res, output, "\n", NULL);
  46         float covered_lines = 0.0;
  47         unsigned int i, total_lines = 0;
  48         bool lines_matter = false;
  49
  50         /*
  51           Output looks like:
  52            File '../../../ccan/tdb2/private.h'
  53            Lines executed:0.00% of 8
  54            /home/ccan/ccan/tdb2/test/run-simple-delete.c:creating 'run-simple-delete.c.gcov'
  55
  56            File '../../../ccan/tdb2/tdb.c'
  57            Lines executed:0.00% of 450
  58         */
  59
  60         for (i = 0; lines[i]; i++) {
  61                 if (strstarts(lines[i], "File '")) {
  62                         char *file = lines[i] + strlen("File '");
  63                         file[strcspn(file, "'")] = '\0';
  64                         if (find_source_file(m, file))
  65                                 lines_matter = true;
  66                         else
  67                                 lines_matter = false;
  68                 } else if (lines_matter
  69                            && strstarts(lines[i], "Lines executed:")) {
  70                         float ex;
  71                         unsigned of;
  72                         if (sscanf(lines[i], "Lines executed:%f%% of %u",
  73                                    &ex, &of) != 2)
  74                                 errx(1, "Could not parse line '%s'", lines[i]);
  75                         total_lines += of;
  76                         covered_lines += ex / 100.0 * of;
  77                 } else if (full_gcov && strstr(lines[i], ":creating '")) {
  78                         char *file, *filename, *apostrophe;
  79                         apostrophe = strchr(lines[i], '\'');
  80                         filename = apostrophe + 1;
  81                         apostrophe = strchr(filename, '\'');
  82                         *apostrophe = '\0';
  83                         if (lines_matter) {
  84                                 file = grab_file(res, filename, NULL);
  85                                 if (!file) {
  86                                         res->what = talloc_asprintf(res,
  87                                                             "Reading %s",
  88                                                             filename);
  89                                         res->output = talloc_strdup(res,
  90                                                             strerror(errno));
  91                                         return;
  92                                 }
  93                                 res->output = talloc_append_string(res->output,
  94                                                                    file);
  95                         }
  96                         if (tools_verbose)
  97                                 printf("Unlinking %s", filename);
  98                         unlink(filename);
  99                 }
 100         }
 101
 102         /* Nothing covered?  We can't tell if there's a source file which
 103          * was never executed, or there really is no code to execute, so
 104          * assume the latter: this test deserves no score. */
 105         if (total_lines == 0) {
 106                 res->uncovered = 1.0;
 107                 run_coverage_tests.total_score = 0;
 108         } else
 109                 res->uncovered = 1.0 - covered_lines / total_lines;
 110 }
 111
 112 static void *do_run_coverage_tests(struct manifest *m,
 113                                    bool keep,
 114                                    unsigned int *timeleft)
 115 {
 116         struct coverage_result *res;
 117         struct ccan_file *i;
 118         char *cmdout;
 119         char *covcmd;
 120         bool ok;
 121         bool full_gcov = (verbose > 1);
 122
 123         res = talloc(m, struct coverage_result);
 124         res->what = NULL;
 125         res->output = talloc_strdup(res, "");
 126         res->uncovered = 1.0;
 127
 128         /* This tells gcov where we put those .gcno files. */
 129         covcmd = talloc_asprintf(m, "gcov %s -o %s",
 130                                  full_gcov ? "" : "-n",
 131                                  talloc_dirname(res, m->info_file->compiled));
 132
 133         /* Run them all. */
 134         list_for_each(&m->run_tests, i, list) {
 135                 cmdout = run_command(m, timeleft, i->cov_compiled);
 136                 if (cmdout) {
 137                         res->what = i->fullname;
 138                         res->output = talloc_steal(res, cmdout);
 139                         return res;
 140                 }
 141                 covcmd = talloc_asprintf_append(covcmd, " %s", i->fullname);
 142         }
 143
 144         list_for_each(&m->api_tests, i, list) {
 145                 cmdout = run_command(m, timeleft, i->cov_compiled);
 146                 if (cmdout) {
 147                         res->what = i->fullname;
 148                         res->output = talloc_steal(res, cmdout);
 149                         return res;
 150                 }
 151                 covcmd = talloc_asprintf_append(covcmd, " %s", i->fullname);
 152         }
 153
 154         /* Now run gcov: we want output even if it succeeds. */
 155         cmdout = run_with_timeout(m, covcmd, &ok, timeleft);
 156         if (!ok) {
 157                 res->what = "Running gcov";
 158                 res->output = talloc_steal(res, cmdout);
 159                 return res;
 160         }
 161
 162         analyze_coverage(m, res, cmdout, full_gcov);
 163
 164         return res;
 165 }
 166
 167 /* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */
 168 static unsigned int score_coverage(struct manifest *m, void *check_result)
 169 {
 170         struct coverage_result *res = check_result;
 171         float thresh;
 172         unsigned int i;
 173
 174         for (i = 0, thresh = 0.5;
 175              i < run_coverage_tests.total_score;
 176              i++, thresh /= 2) {
 177                 if (res->uncovered > thresh)
 178                         break;
 179         }
 180         return i;
 181 }
 182
 183 static const char *describe_run_coverage_tests(struct manifest *m,
 184                                                void *check_result)
 185 {
 186         struct coverage_result *res = check_result;
 187         bool full_gcov = (verbose > 1);
 188         char *ret;
 189
 190         if (res->what)
 191                 return talloc_asprintf(m, "%s: %s", res->what, res->output);
 192
 193         if (!verbose)
 194                 return NULL;
 195
 196         ret = talloc_asprintf(m, "Tests achieved %0.2f%% coverage",
 197                               (1.0 - res->uncovered) * 100);
 198         if (full_gcov)
 199                 ret = talloc_asprintf_append(ret, "\n%s", res->output);
 200         return ret;
 201 }
 202
 203 struct ccanlint run_coverage_tests = {
 204         .key = "test-coverage",
 205         .name = "Code coverage of module tests",
 206         .total_score = 5,
 207         .score = score_coverage,
 208         .check = do_run_coverage_tests,
 209         .describe = describe_run_coverage_tests,
 210 };
 211
 212 REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, &run_tests, NULL);