From: Rusty Russell Date: Fri, 27 Aug 2010 04:15:16 +0000 (+0930) Subject: ccanlint: use gcov to rate test coverage (score out of 5) X-Git-Url: https://git.ozlabs.org/?p=ccan;a=commitdiff_plain;h=a200e1ad1cf5a4828ea4e4e222838ddad5a4a9a3 ccanlint: use gcov to rate test coverage (score out of 5) --- diff --git a/tools/ccanlint/ccanlint.h b/tools/ccanlint/ccanlint.h index ad383028..247d0d0d 100644 --- a/tools/ccanlint/ccanlint.h +++ b/tools/ccanlint/ccanlint.h @@ -139,6 +139,9 @@ struct ccan_file { /* If this file gets compiled (eg. .C file to .o file), result here. */ char *compiled; + + /* Compiled with coverage information. */ + char *cov_compiled; }; /* A new ccan_file, with the given name (talloc_steal onto returned value). */ diff --git a/tools/ccanlint/compulsory_tests/build_objs.c b/tools/ccanlint/compulsory_tests/build_objs.c index 6899b7b0..af7a57b7 100644 --- a/tools/ccanlint/compulsory_tests/build_objs.c +++ b/tools/ccanlint/compulsory_tests/build_objs.c @@ -35,7 +35,7 @@ static void *check_objs_build(struct manifest *m, build_objs.total_score++; i->compiled = maybe_temp_file(m, "", keep, fullfile); - err = compile_object(m, fullfile, ccan_dir, i->compiled); + err = compile_object(m, fullfile, ccan_dir, "", i->compiled); if (err) { talloc_free(i->compiled); if (report) diff --git a/tools/ccanlint/compulsory_tests/check_includes_build.c b/tools/ccanlint/compulsory_tests/check_includes_build.c index b3e4ee68..79d10d29 100644 --- a/tools/ccanlint/compulsory_tests/check_includes_build.c +++ b/tools/ccanlint/compulsory_tests/check_includes_build.c @@ -60,7 +60,7 @@ static void *check_includes_build(struct manifest *m, } close(fd); - return compile_object(m, tmpsrc, ccan_dir, tmpobj); + return compile_object(m, tmpsrc, ccan_dir, "", tmpobj); } static const char *describe_includes_build(struct manifest *m, diff --git a/tools/ccanlint/compulsory_tests/compile_test_helpers.c b/tools/ccanlint/compulsory_tests/compile_test_helpers.c index ac29aa3c..94daaa1b 100644 --- a/tools/ccanlint/compulsory_tests/compile_test_helpers.c +++ b/tools/ccanlint/compulsory_tests/compile_test_helpers.c @@ -26,7 +26,8 @@ static char *compile(struct manifest *m, struct ccan_file *cfile) { cfile->compiled = maybe_temp_file(m, "", keep, cfile->fullname); - return compile_object(m, cfile->fullname, ccan_dir, cfile->compiled); + return compile_object(m, cfile->fullname, ccan_dir, "", + cfile->compiled); } static void *do_compile_test_helpers(struct manifest *m, diff --git a/tools/ccanlint/tests/build-coverage.c b/tools/ccanlint/tests/build-coverage.c new file mode 100644 index 00000000..f1010204 --- /dev/null +++ b/tools/ccanlint/tests/build-coverage.c @@ -0,0 +1,217 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "build-coverage.h" + +/* Note: we already test safe_mode in run_tests.c */ +static const char *can_run_coverage(struct manifest *m) +{ + unsigned int timeleft = default_timeout_ms; + char *output = run_command(m, &timeleft, "gcov -h"); + + if (output) + return talloc_asprintf(m, "No gcov support: %s", output); + return NULL; +} + +static char *build_module_objs_with_coverage(struct manifest *m, bool keep, + char **modobjs) +{ + struct ccan_file *i; + + *modobjs = talloc_strdup(m, ""); + list_for_each(&m->c_files, i, list) { + char *err; + char *fullfile = talloc_asprintf(m, "%s/%s", m->dir, i->name); + + i->cov_compiled = maybe_temp_file(m, "", keep, fullfile); + err = compile_object(m, fullfile, ccan_dir, "", + i->cov_compiled); + if (err) { + talloc_free(i->cov_compiled); + return err; + } + *modobjs = talloc_asprintf_append(*modobjs, + " %s", i->cov_compiled); + } + return NULL; +} + +static char *obj_list(const struct manifest *m, const char *modobjs) +{ + char *list; + struct ccan_file *i; + + /* We expect to be linked with tap, unless that's us. */ + if (!streq(m->basename, "tap")) + list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir); + else + list = talloc_strdup(m, ""); + + /* Objects from any other C files. */ + list_for_each(&m->other_test_c_files, i, list) + list = talloc_asprintf_append(list, " %s", i->compiled); + + if (modobjs) + list = talloc_append_string(list, modobjs); + + /* Other ccan modules (don't need coverage versions of those). */ + list_for_each(&m->dep_dirs, i, list) { + if (i->compiled) + list = talloc_asprintf_append(list, " %s", i->compiled); + } + + return list; +} + +static char *lib_list(const struct manifest *m) +{ + unsigned int i, num; + char **libs = get_libs(m, ".", &num, &m->info_file->compiled); + char *ret = talloc_strdup(m, ""); + + for (i = 0; i < num; i++) + ret = talloc_asprintf_append(ret, "-l%s ", libs[i]); + return ret; +} + +/* Grrr... gcov drops a turd in the current directory. */ +void move_gcov_turd(const char *dir, + struct ccan_file *file, const char *extension) +{ + char *base, *gcovfile, *gcovdest; + + base = talloc_basename(file, file->name); + gcovfile = talloc_asprintf(file, "%s/%.*s%s", + dir, strlen(base)-2, base, extension); + gcovdest = talloc_asprintf(file, "%s/%.*s%s", + talloc_dirname(base, file->cov_compiled), + strlen(base)-2, base, extension); + if (!move_file(gcovfile, gcovdest)) + err(1, "Could not move %s to %s", gcovfile, gcovdest); + talloc_free(base); +} + +static char *cov_compile(const void *ctx, + struct manifest *m, + struct ccan_file *file, + const char *modobjs, + bool keep) +{ + char *errmsg; + char path[PATH_MAX]; + + file->cov_compiled = maybe_temp_file(ctx, "", keep, file->fullname); + errmsg = compile_and_link(ctx, file->fullname, ccan_dir, + obj_list(m, modobjs), + COVERAGE_CFLAGS, + lib_list(m), file->cov_compiled); + if (errmsg) { + talloc_free(file->cov_compiled); + return errmsg; + } + + move_gcov_turd(getcwd(path, sizeof(path)), file, ".gcno"); + return NULL; +} + +struct compile_tests_result { + struct list_node list; + const char *filename; + const char *description; + const char *output; +}; + +static void *do_compile_coverage_tests(struct manifest *m, + bool keep, + unsigned int *timeleft) +{ + struct list_head *list = talloc(m, struct list_head); + char *cmdout, *modobjs = NULL; + struct ccan_file *i; + struct compile_tests_result *res; + + list_head_init(list); + + if (!list_empty(&m->api_tests)) { + cmdout = build_module_objs_with_coverage(m, keep, &modobjs); + if (cmdout) { + res = talloc(list, struct compile_tests_result); + res->filename = "Module objects with coverage"; + res->description = "failed to compile"; + res->output = talloc_steal(res, cmdout); + list_add_tail(list, &res->list); + return list; + } + } + + list_for_each(&m->run_tests, i, list) { + compile_tests.total_score++; + cmdout = cov_compile(m, m, i, NULL, keep); + if (cmdout) { + res = talloc(list, struct compile_tests_result); + res->filename = i->name; + res->description = "failed to compile"; + res->output = talloc_steal(res, cmdout); + list_add_tail(list, &res->list); + } + } + + list_for_each(&m->api_tests, i, list) { + compile_tests.total_score++; + cmdout = cov_compile(m, m, i, modobjs, keep); + if (cmdout) { + res = talloc(list, struct compile_tests_result); + res->filename = i->name; + res->description = "failed to compile"; + res->output = talloc_steal(res, cmdout); + list_add_tail(list, &res->list); + } + } + + if (list_empty(list)) { + talloc_free(list); + list = NULL; + } + + return list; +} + +static const char *describe_compile_coverage_tests(struct manifest *m, + void *check_result) +{ + struct list_head *list = check_result; + struct compile_tests_result *i; + char *descrip; + + descrip = talloc_strdup(list, + "Compilation of tests for coverage failed:\n"); + + list_for_each(list, i, list) + descrip = talloc_asprintf_append(descrip, "%s %s\n%s", + i->filename, i->description, + i->output); + return descrip; +} + +struct ccanlint compile_coverage_tests = { + .key = "compile-coverage-tests", + .name = "Module tests compile with profiling", + .check = do_compile_coverage_tests, + .describe = describe_compile_coverage_tests, + .can_run = can_run_coverage, +}; + +REGISTER_TEST(compile_coverage_tests, &compile_tests, NULL); diff --git a/tools/ccanlint/tests/build-coverage.h b/tools/ccanlint/tests/build-coverage.h new file mode 100644 index 00000000..0bbac2be --- /dev/null +++ b/tools/ccanlint/tests/build-coverage.h @@ -0,0 +1,8 @@ +#ifndef CCANLINT_BUILD_COVERAGE_H +#define CCANLINT_BUILD_COVERAGE_H + +/* FIXME: gcov dumps a file into a random dir. */ +void move_gcov_turd(const char *dir, + struct ccan_file *file, const char *extension); + +#endif /* CCANLINT_BUILD_COVERAGE_H */ diff --git a/tools/ccanlint/tests/run-coverage.c b/tools/ccanlint/tests/run-coverage.c new file mode 100644 index 00000000..74247e32 --- /dev/null +++ b/tools/ccanlint/tests/run-coverage.c @@ -0,0 +1,184 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "build-coverage.h" + +struct coverage_result { + float uncovered; + const char *what; + const char *output; +}; + +static bool find_source_file(struct manifest *m, const char *filename) +{ + struct ccan_file *i; + + list_for_each(&m->c_files, i, list) { + if (streq(i->fullname, filename)) + return true; + } + list_for_each(&m->h_files, i, list) { + if (streq(i->fullname, filename)) + return true; + } + return false; +} + +/* FIXME: Don't know how stable this is. Read cov files directly? */ +static void analyze_coverage(struct manifest *m, + struct coverage_result *res, const char *output) +{ + char **lines = strsplit(res, output, "\n", NULL); + float covered_lines = 0.0; + unsigned int i, total_lines = 0; + bool lines_matter = false; + + /* FIXME: We assume GCOV mentions all files! + Output looks like: + File '../../../ccan/tdb2/private.h' + Lines executed:0.00% of 8 + + File '../../../ccan/tdb2/tdb.c' + Lines executed:0.00% of 450 + */ + + for (i = 0; lines[i]; i++) { + if (strstarts(lines[i], "File '")) { + char *file = lines[i] + strlen("File '"); + file[strcspn(file, "'")] = '\0'; + if (find_source_file(m, file)) + lines_matter = true; + else + lines_matter = false; + } else if (lines_matter + && strstarts(lines[i], "Lines executed:")) { + float ex; + unsigned of; + if (sscanf(lines[i], "Lines executed:%f%% of %u", + &ex, &of) != 2) + errx(1, "Could not parse line '%s'", lines[i]); + total_lines += of; + covered_lines += ex / 100.0 * of; + } + } + + /* Nothing covered? */ + if (total_lines == 0) + res->uncovered = 1.0; + else + res->uncovered = 1.0 - covered_lines / total_lines; +} + +static void *do_run_coverage_tests(struct manifest *m, + bool keep, + unsigned int *timeleft) +{ + struct coverage_result *res; + struct ccan_file *i; + char *cmdout; + char *olddir; + char *covcmd; + bool ok; + + /* We run tests in the module directory, so any paths + * referenced can all be module-local. */ + olddir = talloc_getcwd(m); + if (!olddir) + err(1, "Could not save cwd"); + if (chdir(m->dir) != 0) + err(1, "Could not chdir to %s", m->dir); + + res = talloc(m, struct coverage_result); + res->what = NULL; + res->uncovered = 1.0; + + /* This tells gcov where we put those .gcno files. */ + covcmd = talloc_asprintf(m, "gcov -n -o %s", + talloc_dirname(res, m->info_file->compiled)); + + /* Run them all. */ + list_for_each(&m->run_tests, i, list) { + cmdout = run_command(m, timeleft, i->cov_compiled); + if (cmdout) { + res->what = i->fullname; + res->output = talloc_steal(res, cmdout); + return res; + } + covcmd = talloc_asprintf_append(covcmd, " %s", i->name); + move_gcov_turd(olddir, i, ".gcda"); + } + + list_for_each(&m->api_tests, i, list) { + cmdout = run_command(m, timeleft, i->cov_compiled); + if (cmdout) { + res->what = i->fullname; + res->output = talloc_steal(res, cmdout); + return res; + } + covcmd = talloc_asprintf_append(covcmd, " %s", i->name); + move_gcov_turd(olddir, i, ".gcda"); + } + + /* Now run gcov: we want output even if it succeeds. */ + cmdout = run_with_timeout(m, covcmd, &ok, timeleft); + if (!ok) { + res->what = "Running gcov"; + res->output = talloc_steal(res, cmdout); + return res; + } + + analyze_coverage(m, res, cmdout); + return res; +} + +/* 1 point for 50%, 2 points for 75%, 3 points for 87.5%... */ +static unsigned int score_coverage(struct manifest *m, void *check_result) +{ + struct coverage_result *res = check_result; + float thresh; + unsigned int i; + + for (i = 0, thresh = 0.5; + i < run_coverage_tests.total_score; + i++, thresh /= 2) { + if (res->uncovered > thresh) + break; + } + return i; +} + +static const char *describe_run_coverage_tests(struct manifest *m, + void *check_result) +{ + struct coverage_result *res = check_result; + + if (res->what) + return talloc_asprintf(m, "%s: %s", res->what, res->output); + + return talloc_asprintf(m, "Tests achieved %0.2f%% coverage", + (1.0 - res->uncovered) * 100); +} + +struct ccanlint run_coverage_tests = { + .key = "test-coverage", + .name = "Code coverage of module tests", + .total_score = 5, + .score = score_coverage, + .check = do_run_coverage_tests, + .describe = describe_run_coverage_tests, +}; + +REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL); diff --git a/tools/compile.c b/tools/compile.c index fc7dc347..bf68ad26 100644 --- a/tools/compile.c +++ b/tools/compile.c @@ -17,10 +17,11 @@ char *link_objects(const void *ctx, const char *objs, char **errmsg) /* Compile a single C file to an object file. Returns errmsg if fails. */ char *compile_object(const void *ctx, const char *cfile, const char *ccandir, + const char *extra_cflags, const char *outfile) { - return run_command(ctx, NULL, "cc " CFLAGS " -I%s -c -o %s %s", - ccandir, outfile, cfile); + return run_command(ctx, NULL, "cc " CFLAGS " -I%s %s -c -o %s %s", + ccandir, extra_cflags, outfile, cfile); } /* Compile and link single C file, with object files. diff --git a/tools/tools.c b/tools/tools.c index 8ce193cd..7de02e3b 100644 --- a/tools/tools.c +++ b/tools/tools.c @@ -63,10 +63,8 @@ static void killme(int sig) kill(-getpid(), SIGKILL); } -static char *run_with_timeout(const void *ctx, - const char *cmd, - bool *ok, - unsigned *timeout_ms) +char *run_with_timeout(const void *ctx, const char *cmd, + bool *ok, unsigned *timeout_ms) { pid_t pid; int p[2]; diff --git a/tools/tools.h b/tools/tools.h index babd30c9..35f6bd8c 100644 --- a/tools/tools.h +++ b/tools/tools.h @@ -9,7 +9,9 @@ #define SPACE_CHARS " \f\n\r\t\v" /* FIXME: Nested functions break with -Wmissing-prototypes -Wmissing-declarations */ -#define CFLAGS "-g -Wall -Wundef -Wstrict-prototypes -Wold-style-definition -Werror -I../.." +#define CFLAGS "-g -Wall -Wundef -Wstrict-prototypes -Wold-style-definition -Werror" + +#define COVERAGE_CFLAGS "-fprofile-arcs -ftest-coverage" /* This actually compiles and runs the info file to get dependencies. */ char **get_deps(const void *ctx, const char *dir, bool recurse, @@ -28,6 +30,8 @@ char *talloc_basename(const void *ctx, const char *dir); char *talloc_dirname(const void *ctx, const char *dir); char *talloc_getcwd(const void *ctx); char *run_command(const void *ctx, unsigned int *time_ms, const char *fmt, ...); +char *run_with_timeout(const void *ctx, const char *cmd, + bool *ok, unsigned *timeout_ms); char *temp_file(const void *ctx, const char *extension); bool move_file(const char *oldname, const char *newname); @@ -40,6 +44,7 @@ bool move_file(const char *oldname, const char *newname); char *link_objects(const void *ctx, const char *objs, char **errmsg); /* Compile a single C file to an object file. Returns errmsg if fails. */ char *compile_object(const void *ctx, const char *cfile, const char *ccandir, + const char *extra_cflags, const char *outfile); /* Compile and link single C file, with object files, libs, etc. NULL on * success, error output on fail. */