ccanlint: use gcov to rate test coverage (score out of 5)
authorRusty Russell <rusty@rustcorp.com.au>
Fri, 27 Aug 2010 04:15:16 +0000 (13:45 +0930)
committerRusty Russell <rusty@rustcorp.com.au>
Fri, 27 Aug 2010 04:15:16 +0000 (13:45 +0930)
tools/ccanlint/ccanlint.h
tools/ccanlint/compulsory_tests/build_objs.c
tools/ccanlint/compulsory_tests/check_includes_build.c
tools/ccanlint/compulsory_tests/compile_test_helpers.c
tools/ccanlint/tests/build-coverage.c [new file with mode: 0644]
tools/ccanlint/tests/build-coverage.h [new file with mode: 0644]
tools/ccanlint/tests/run-coverage.c [new file with mode: 0644]
tools/compile.c
tools/tools.c
tools/tools.h

index ad38302861046cdab7475cbef03f22365b51db0f..247d0d0d12cbe92c18f22feb6ee09565bbd1a5e3 100644 (file)
@@ -139,6 +139,9 @@ struct ccan_file {
 
        /* If this file gets compiled (eg. .C file to .o file), result here. */
        char *compiled;
+
+       /* Compiled with coverage information. */
+       char *cov_compiled;
 };
 
 /* A new ccan_file, with the given name (talloc_steal onto returned value). */
index 6899b7b04986e5418e03f20e861a9435b3cea413..af7a57b73e8482c18f627abfe5d2e0e814222764 100644 (file)
@@ -35,7 +35,7 @@ static void *check_objs_build(struct manifest *m,
                build_objs.total_score++;
 
                i->compiled = maybe_temp_file(m, "", keep, fullfile);
-               err = compile_object(m, fullfile, ccan_dir, i->compiled);
+               err = compile_object(m, fullfile, ccan_dir, "", i->compiled);
                if (err) {
                        talloc_free(i->compiled);
                        if (report)
index b3e4ee682b87ce1db21cbfd9654229f80c796caa..79d10d29775d65c7fa3a7e2695c113429ff17d35 100644 (file)
@@ -60,7 +60,7 @@ static void *check_includes_build(struct manifest *m,
        }
        close(fd);
 
-       return compile_object(m, tmpsrc, ccan_dir, tmpobj);
+       return compile_object(m, tmpsrc, ccan_dir, "", tmpobj);
 }
 
 static const char *describe_includes_build(struct manifest *m,
index ac29aa3cd2799ed55bb24281fc0414b89df8815e..94daaa1bf4bbe1128d234cf767e228f08bd1680b 100644 (file)
@@ -26,7 +26,8 @@ static char *compile(struct manifest *m,
                     struct ccan_file *cfile)
 {
        cfile->compiled = maybe_temp_file(m, "", keep, cfile->fullname);
-       return compile_object(m, cfile->fullname, ccan_dir, cfile->compiled);
+       return compile_object(m, cfile->fullname, ccan_dir, "",
+                             cfile->compiled);
 }
 
 static void *do_compile_test_helpers(struct manifest *m,
diff --git a/tools/ccanlint/tests/build-coverage.c b/tools/ccanlint/tests/build-coverage.c
new file mode 100644 (file)
index 0000000..f101020
--- /dev/null
@@ -0,0 +1,217 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+#include "build-coverage.h"
+
+/* Note: we already test safe_mode in run_tests.c */
+static const char *can_run_coverage(struct manifest *m)
+{
+       unsigned int timeleft = default_timeout_ms;
+       char *output = run_command(m, &timeleft, "gcov -h");
+
+       if (output)
+               return talloc_asprintf(m, "No gcov support: %s", output);
+       return NULL;
+}
+
+static char *build_module_objs_with_coverage(struct manifest *m, bool keep,
+                                            char **modobjs)
+{
+       struct ccan_file *i;
+
+       *modobjs = talloc_strdup(m, "");
+       list_for_each(&m->c_files, i, list) {
+               char *err;
+               char *fullfile = talloc_asprintf(m, "%s/%s", m->dir, i->name);
+
+               i->cov_compiled = maybe_temp_file(m, "", keep, fullfile);
+               err = compile_object(m, fullfile, ccan_dir, "",
+                                    i->cov_compiled);
+               if (err) {
+                       talloc_free(i->cov_compiled);
+                       return err;
+               }
+               *modobjs = talloc_asprintf_append(*modobjs,
+                                                 " %s", i->cov_compiled);
+       }
+       return NULL;
+}
+
+static char *obj_list(const struct manifest *m, const char *modobjs)
+{
+       char *list;
+       struct ccan_file *i;
+
+       /* We expect to be linked with tap, unless that's us. */
+       if (!streq(m->basename, "tap"))
+               list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir);
+       else
+               list = talloc_strdup(m, "");
+
+       /* Objects from any other C files. */
+       list_for_each(&m->other_test_c_files, i, list)
+               list = talloc_asprintf_append(list, " %s", i->compiled);
+
+       if (modobjs)
+               list = talloc_append_string(list, modobjs);
+
+       /* Other ccan modules (don't need coverage versions of those). */
+       list_for_each(&m->dep_dirs, i, list) {
+               if (i->compiled)
+                       list = talloc_asprintf_append(list, " %s", i->compiled);
+       }
+
+       return list;
+}
+
+static char *lib_list(const struct manifest *m)
+{
+       unsigned int i, num;
+       char **libs = get_libs(m, ".", &num, &m->info_file->compiled);
+       char *ret = talloc_strdup(m, "");
+
+       for (i = 0; i < num; i++)
+               ret = talloc_asprintf_append(ret, "-l%s ", libs[i]);
+       return ret;
+}
+
+/* Grrr... gcov drops a turd in the current directory. */
+void move_gcov_turd(const char *dir,
+                   struct ccan_file *file, const char *extension)
+{
+       char *base, *gcovfile, *gcovdest;
+
+       base = talloc_basename(file, file->name);
+       gcovfile = talloc_asprintf(file, "%s/%.*s%s",
+                                  dir, strlen(base)-2, base, extension);
+       gcovdest = talloc_asprintf(file, "%s/%.*s%s",
+                                  talloc_dirname(base, file->cov_compiled),
+                                  strlen(base)-2, base, extension);
+       if (!move_file(gcovfile, gcovdest))
+               err(1, "Could not move %s to %s", gcovfile, gcovdest);
+       talloc_free(base);
+}
+
+static char *cov_compile(const void *ctx,
+                        struct manifest *m,
+                        struct ccan_file *file,
+                        const char *modobjs,
+                        bool keep)
+{
+       char *errmsg;
+       char path[PATH_MAX];
+
+       file->cov_compiled = maybe_temp_file(ctx, "", keep, file->fullname);
+       errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
+                                 obj_list(m, modobjs),
+                                 COVERAGE_CFLAGS,
+                                 lib_list(m), file->cov_compiled);
+       if (errmsg) {
+               talloc_free(file->cov_compiled);
+               return errmsg;
+       }
+
+       move_gcov_turd(getcwd(path, sizeof(path)), file, ".gcno");
+       return NULL;
+}
+
+struct compile_tests_result {
+       struct list_node list;
+       const char *filename;
+       const char *description;
+       const char *output;
+};
+
+static void *do_compile_coverage_tests(struct manifest *m,
+                                      bool keep,
+                                      unsigned int *timeleft)
+{
+       struct list_head *list = talloc(m, struct list_head);
+       char *cmdout, *modobjs = NULL;
+       struct ccan_file *i;
+       struct compile_tests_result *res;
+
+       list_head_init(list);
+
+       if (!list_empty(&m->api_tests)) {
+               cmdout = build_module_objs_with_coverage(m, keep, &modobjs);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = "Module objects with coverage";
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+                       return list;
+               }
+       }
+
+       list_for_each(&m->run_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = cov_compile(m, m, i, NULL, keep);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       list_for_each(&m->api_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = cov_compile(m, m, i, modobjs, keep);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       if (list_empty(list)) {
+               talloc_free(list);
+               list = NULL;
+       }
+
+       return list;
+}
+
+static const char *describe_compile_coverage_tests(struct manifest *m,
+                                                  void *check_result)
+{
+       struct list_head *list = check_result;
+       struct compile_tests_result *i;
+       char *descrip;
+
+       descrip = talloc_strdup(list,
+                               "Compilation of tests for coverage failed:\n");
+
+       list_for_each(list, i, list)
+               descrip = talloc_asprintf_append(descrip, "%s %s\n%s",
+                                                i->filename, i->description,
+                                                i->output);
+       return descrip;
+}
+
+struct ccanlint compile_coverage_tests = {
+       .key = "compile-coverage-tests",
+       .name = "Module tests compile with profiling",
+       .check = do_compile_coverage_tests,
+       .describe = describe_compile_coverage_tests,
+       .can_run = can_run_coverage,
+};
+
+REGISTER_TEST(compile_coverage_tests, &compile_tests, NULL);
diff --git a/tools/ccanlint/tests/build-coverage.h b/tools/ccanlint/tests/build-coverage.h
new file mode 100644 (file)
index 0000000..0bbac2b
--- /dev/null
@@ -0,0 +1,8 @@
+#ifndef CCANLINT_BUILD_COVERAGE_H
+#define CCANLINT_BUILD_COVERAGE_H
+
+/* FIXME: gcov dumps a file into a random dir. */
+void move_gcov_turd(const char *dir,
+                   struct ccan_file *file, const char *extension);
+
+#endif /* CCANLINT_BUILD_COVERAGE_H */
diff --git a/tools/ccanlint/tests/run-coverage.c b/tools/ccanlint/tests/run-coverage.c
new file mode 100644 (file)
index 0000000..74247e3
--- /dev/null
@@ -0,0 +1,184 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str_talloc/str_talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+#include "build-coverage.h"
+
+struct coverage_result {
+       float uncovered;
+       const char *what;
+       const char *output;
+};
+
+static bool find_source_file(struct manifest *m, const char *filename)
+{
+       struct ccan_file *i;
+
+       list_for_each(&m->c_files, i, list) {
+               if (streq(i->fullname, filename))
+                       return true;
+       }
+       list_for_each(&m->h_files, i, list) {
+               if (streq(i->fullname, filename))
+                       return true;
+       }
+       return false;
+}
+
+/* FIXME: Don't know how stable this is.  Read cov files directly? */
+static void analyze_coverage(struct manifest *m,
+                            struct coverage_result *res, const char *output)
+{
+       char **lines = strsplit(res, output, "\n", NULL);
+       float covered_lines = 0.0;
+       unsigned int i, total_lines = 0;
+       bool lines_matter = false;
+
+       /* FIXME: We assume GCOV mentions all files!
+         Output looks like:
+          File '../../../ccan/tdb2/private.h'
+          Lines executed:0.00% of 8
+
+          File '../../../ccan/tdb2/tdb.c'
+          Lines executed:0.00% of 450
+       */
+
+       for (i = 0; lines[i]; i++) {
+               if (strstarts(lines[i], "File '")) {
+                       char *file = lines[i] + strlen("File '");
+                       file[strcspn(file, "'")] = '\0';
+                       if (find_source_file(m, file))
+                               lines_matter = true;
+                       else
+                               lines_matter = false;
+               } else if (lines_matter
+                          && strstarts(lines[i], "Lines executed:")) {
+                       float ex;
+                       unsigned of;
+                       if (sscanf(lines[i], "Lines executed:%f%% of %u",
+                                  &ex, &of) != 2)
+                               errx(1, "Could not parse line '%s'", lines[i]);
+                       total_lines += of;
+                       covered_lines += ex / 100.0 * of;
+               }
+       }
+
+       /* Nothing covered? */
+       if (total_lines == 0)
+               res->uncovered = 1.0;
+       else
+               res->uncovered = 1.0 - covered_lines / total_lines;
+}
+
+static void *do_run_coverage_tests(struct manifest *m,
+                                  bool keep,
+                                  unsigned int *timeleft)
+{
+       struct coverage_result *res;
+       struct ccan_file *i;
+       char *cmdout;
+       char *olddir;
+       char *covcmd;
+       bool ok;
+
+       /* We run tests in the module directory, so any paths
+        * referenced can all be module-local. */
+       olddir = talloc_getcwd(m);
+       if (!olddir)
+               err(1, "Could not save cwd");
+       if (chdir(m->dir) != 0)
+               err(1, "Could not chdir to %s", m->dir);
+
+       res = talloc(m, struct coverage_result);
+       res->what = NULL;
+       res->uncovered = 1.0;
+
+       /* This tells gcov where we put those .gcno files. */
+       covcmd = talloc_asprintf(m, "gcov -n -o %s",
+                                talloc_dirname(res, m->info_file->compiled));
+
+       /* Run them all. */
+       list_for_each(&m->run_tests, i, list) {
+               cmdout = run_command(m, timeleft, i->cov_compiled);
+               if (cmdout) {
+                       res->what = i->fullname;
+                       res->output = talloc_steal(res, cmdout);
+                       return res;
+               }
+               covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
+               move_gcov_turd(olddir, i, ".gcda");
+       }
+
+       list_for_each(&m->api_tests, i, list) {
+               cmdout = run_command(m, timeleft, i->cov_compiled);
+               if (cmdout) {
+                       res->what = i->fullname;
+                       res->output = talloc_steal(res, cmdout);
+                       return res;
+               }
+               covcmd = talloc_asprintf_append(covcmd, " %s", i->name);
+               move_gcov_turd(olddir, i, ".gcda");
+       }
+
+       /* Now run gcov: we want output even if it succeeds. */
+       cmdout = run_with_timeout(m, covcmd, &ok, timeleft);
+       if (!ok) {
+               res->what = "Running gcov";
+               res->output = talloc_steal(res, cmdout);
+               return res;
+       }
+
+       analyze_coverage(m, res, cmdout);
+       return res;
+}
+
+/* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */
+static unsigned int score_coverage(struct manifest *m, void *check_result)
+{
+       struct coverage_result *res = check_result;
+       float thresh;
+       unsigned int i;
+
+       for (i = 0, thresh = 0.5;
+            i < run_coverage_tests.total_score;
+            i++, thresh /= 2) {
+               if (res->uncovered > thresh)
+                       break;
+       }
+       return i;
+}
+
+static const char *describe_run_coverage_tests(struct manifest *m,
+                                              void *check_result)
+{
+       struct coverage_result *res = check_result;
+
+       if (res->what)
+               return talloc_asprintf(m, "%s: %s", res->what, res->output);
+
+       return talloc_asprintf(m, "Tests achieved %0.2f%% coverage",
+                              (1.0 - res->uncovered) * 100);
+}
+
+struct ccanlint run_coverage_tests = {
+       .key = "test-coverage",
+       .name = "Code coverage of module tests",
+       .total_score = 5,
+       .score = score_coverage,
+       .check = do_run_coverage_tests,
+       .describe = describe_run_coverage_tests,
+};
+
+REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);
index fc7dc34789fbcfac6c832fce41ff7b6e0253499c..bf68ad26477644dc8542aaa1414c9d10fda1572b 100644 (file)
@@ -17,10 +17,11 @@ char *link_objects(const void *ctx, const char *objs, char **errmsg)
 
 /* Compile a single C file to an object file.  Returns errmsg if fails. */
 char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
+                    const char *extra_cflags,
                     const char *outfile)
 {
-       return run_command(ctx, NULL, "cc " CFLAGS " -I%s -c -o %s %s",
-                          ccandir, outfile, cfile);
+       return run_command(ctx, NULL, "cc " CFLAGS " -I%s %s -c -o %s %s",
+                          ccandir, extra_cflags, outfile, cfile);
 }
 
 /* Compile and link single C file, with object files.
index 8ce193cd226bd0905f7a6738c4f70eda4057fe1c..7de02e3b9dfdcc2ae1604d6db5c6d46a1466ecc7 100644 (file)
@@ -63,10 +63,8 @@ static void killme(int sig)
        kill(-getpid(), SIGKILL);
 }
 
-static char *run_with_timeout(const void *ctx,
-                             const char *cmd,
-                             bool *ok,
-                             unsigned *timeout_ms)
+char *run_with_timeout(const void *ctx, const char *cmd,
+                      bool *ok, unsigned *timeout_ms)
 {
        pid_t pid;
        int p[2];
index babd30c9370892c2f6961f3546bf493d94f1ec43..35f6bd8c80dcf3ac31f1b4d2f7d7d765cca31bd6 100644 (file)
@@ -9,7 +9,9 @@
 #define SPACE_CHARS    " \f\n\r\t\v"
 
 /* FIXME: Nested functions break with -Wmissing-prototypes -Wmissing-declarations */
-#define CFLAGS "-g -Wall -Wundef -Wstrict-prototypes -Wold-style-definition -Werror -I../.."
+#define CFLAGS "-g -Wall -Wundef -Wstrict-prototypes -Wold-style-definition -Werror"
+
+#define COVERAGE_CFLAGS "-fprofile-arcs -ftest-coverage"
 
 /* This actually compiles and runs the info file to get dependencies. */
 char **get_deps(const void *ctx, const char *dir, bool recurse,
@@ -28,6 +30,8 @@ char *talloc_basename(const void *ctx, const char *dir);
 char *talloc_dirname(const void *ctx, const char *dir);
 char *talloc_getcwd(const void *ctx);
 char *run_command(const void *ctx, unsigned int *time_ms, const char *fmt, ...);
+char *run_with_timeout(const void *ctx, const char *cmd,
+                      bool *ok, unsigned *timeout_ms);
 char *temp_file(const void *ctx, const char *extension);
 bool move_file(const char *oldname, const char *newname);
 
@@ -40,6 +44,7 @@ bool move_file(const char *oldname, const char *newname);
 char *link_objects(const void *ctx, const char *objs, char **errmsg);
 /* Compile a single C file to an object file.  Returns errmsg if fails. */
 char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
+                    const char *extra_cflags,
                     const char *outfile);
 /* Compile and link single C file, with object files, libs, etc.  NULL on
  * success, error output on fail. */