ccanlint: make tests non-compulsory, always print score.
authorRusty Russell <rusty@rustcorp.com.au>
Wed, 3 Nov 2010 12:55:57 +0000 (23:25 +1030)
committerRusty Russell <rusty@rustcorp.com.au>
Wed, 3 Nov 2010 12:55:57 +0000 (23:25 +1030)
No longer abort ccanlint because a test fails, but you will get a low
score if there are no tests.

(Note: total_score is filled in, even if it's overridden when the test is
 run.  This means that if the prereq fails, that total counts so there is
 still a penalty!)

13 files changed:
tools/ccanlint/ccanlint.c
tools/ccanlint/compulsory_tests/compile_test_helpers.c [deleted file]
tools/ccanlint/compulsory_tests/compile_tests.c [deleted file]
tools/ccanlint/compulsory_tests/has_tests.c [deleted file]
tools/ccanlint/compulsory_tests/run_tests.c [deleted file]
tools/ccanlint/tests/build-coverage.c
tools/ccanlint/tests/compile_test_helpers.c [new file with mode: 0644]
tools/ccanlint/tests/compile_tests.c [new file with mode: 0644]
tools/ccanlint/tests/examples_compile.c
tools/ccanlint/tests/has_tests.c [new file with mode: 0644]
tools/ccanlint/tests/run-coverage.c
tools/ccanlint/tests/run_tests.c [new file with mode: 0644]
tools/ccanlint/tests/run_tests_valgrind.c

index ab805806144670e29dc01e05db63c80706c8030c..01a94758f5916847b2dd9a4c8a86aac89982b75f 100644 (file)
@@ -407,12 +407,13 @@ int main(int argc, char *argv[])
                    talloc_asprintf(m, "%s/test", temp_dir(NULL))) != 0)
                err(1, "Creating test symlink in %s", temp_dir(NULL));
 
-       /* If you don't pass the compulsory tests, you don't even get a score */
+       /* If you don't pass the compulsory tests, you get a score of 0. */
        if (verbose)
                printf("Compulsory tests:\n");
 
        while ((i = get_next_test(&compulsory_tests)) != NULL) {
                if (!run_test(i, summary, &score, &total_score, m)) {
+                       printf("%sTotal score: 0/%u\n", prefix, total_score);
                        errx(1, "%s%s failed", prefix, i->name);
                }
        }
diff --git a/tools/ccanlint/compulsory_tests/compile_test_helpers.c b/tools/ccanlint/compulsory_tests/compile_test_helpers.c
deleted file mode 100644 (file)
index 207c0b2..0000000
+++ /dev/null
@@ -1,67 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_build(struct manifest *m)
-{
-       if (safe_mode)
-               return "Safe mode enabled";
-       return NULL;
-}
-
-static char *compile(struct manifest *m, 
-                    bool keep,
-                    struct ccan_file *cfile)
-{
-       cfile->compiled = maybe_temp_file(m, ".o", keep, cfile->fullname);
-       return compile_object(m, cfile->fullname, ccan_dir, "",
-                             cfile->compiled);
-}
-
-static void *do_compile_test_helpers(struct manifest *m,
-                                    bool keep,
-                                    unsigned int *timeleft)
-{
-       char *cmdout = NULL;
-       struct ccan_file *i;
-
-       list_for_each(&m->other_test_c_files, i, list) {
-               compile_tests.total_score++;
-               cmdout = compile(m, keep, i);
-               if (cmdout)
-                       return talloc_asprintf(m,
-                                              "Failed to compile helper C"
-                                              " code file %s:\n%s",
-                                              i->name, cmdout);
-       }
-       return NULL;
-}
-
-static const char *describe_compile_test_helpers(struct manifest *m,
-                                                void *check_result)
-{
-       return check_result;
-}
-
-struct ccanlint compile_test_helpers = {
-       .key = "compile-helpers",
-       .name = "Module test helper objects compile",
-       .total_score = 1,
-       .check = do_compile_test_helpers,
-       .describe = describe_compile_test_helpers,
-       .can_run = can_build,
-};
-
-REGISTER_TEST(compile_test_helpers, &depends_built);
diff --git a/tools/ccanlint/compulsory_tests/compile_tests.c b/tools/ccanlint/compulsory_tests/compile_tests.c
deleted file mode 100644 (file)
index 1559bd1..0000000
+++ /dev/null
@@ -1,204 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_build(struct manifest *m)
-{
-       if (safe_mode)
-               return "Safe mode enabled";
-       return NULL;
-}
-
-static char *obj_list(const struct manifest *m, bool link_with_module)
-{
-       char *list;
-       struct ccan_file *i;
-
-       /* We expect to be linked with tap, unless that's us. */
-       if (!streq(m->basename, "tap"))
-               list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir);
-       else
-               list = talloc_strdup(m, "");
-
-       /* Objects from any other C files. */
-       list_for_each(&m->other_test_c_files, i, list)
-               list = talloc_asprintf_append(list, " %s", i->compiled);
-
-       /* Our own object files. */
-       if (link_with_module)
-               list_for_each(&m->c_files, i, list)
-                       list = talloc_asprintf_append(list, " %s", i->compiled);
-
-       /* Other ccan modules. */
-       list_for_each(&m->dep_dirs, i, list) {
-               if (i->compiled)
-                       list = talloc_asprintf_append(list, " %s", i->compiled);
-       }
-
-       return list;
-}
-
-static char *lib_list(const struct manifest *m)
-{
-       unsigned int i, num;
-       char **libs = get_libs(m, ".", &num, &m->info_file->compiled);
-       char *ret = talloc_strdup(m, "");
-
-       for (i = 0; i < num; i++)
-               ret = talloc_asprintf_append(ret, "-l%s ", libs[i]);
-       return ret;
-}
-
-static char *compile(const void *ctx,
-                    struct manifest *m,
-                    struct ccan_file *file,
-                    bool fail,
-                    bool link_with_module,
-                    bool keep)
-{
-       char *errmsg;
-
-       file->compiled = maybe_temp_file(ctx, "", keep, file->fullname);
-       errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
-                                 obj_list(m, link_with_module),
-                                 fail ? "-DFAIL" : "",
-                                 lib_list(m), file->compiled);
-       if (errmsg) {
-               talloc_free(file->compiled);
-               return errmsg;
-       }
-       return NULL;
-}
-
-struct compile_tests_result {
-       struct list_node list;
-       const char *filename;
-       const char *description;
-       const char *output;
-};
-
-static void *do_compile_tests(struct manifest *m,
-                             bool keep,
-                             unsigned int *timeleft)
-{
-       struct list_head *list = talloc(m, struct list_head);
-       char *cmdout;
-       struct ccan_file *i;
-       struct compile_tests_result *res;
-
-       list_head_init(list);
-
-       list_for_each(&m->compile_ok_tests, i, list) {
-               compile_tests.total_score++;
-               cmdout = compile(list, m, i, false, false, keep);
-               if (cmdout) {
-                       res = talloc(list, struct compile_tests_result);
-                       res->filename = i->name;
-                       res->description = "failed to compile";
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
-       }
-
-       list_for_each(&m->run_tests, i, list) {
-               compile_tests.total_score++;
-               cmdout = compile(m, m, i, false, false, keep);
-               if (cmdout) {
-                       res = talloc(list, struct compile_tests_result);
-                       res->filename = i->name;
-                       res->description = "failed to compile";
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
-       }
-
-       list_for_each(&m->api_tests, i, list) {
-               compile_tests.total_score++;
-               cmdout = compile(m, m, i, false, true, keep);
-               if (cmdout) {
-                       res = talloc(list, struct compile_tests_result);
-                       res->filename = i->name;
-                       res->description = "failed to compile";
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
-       }
-
-       list_for_each(&m->compile_fail_tests, i, list) {
-               compile_tests.total_score++;
-               cmdout = compile(list, m, i, false, false, false);
-               if (cmdout) {
-                       res = talloc(list, struct compile_tests_result);
-                       res->filename = i->name;
-                       res->description = "failed to compile without -DFAIL";
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               } else {
-                       cmdout = compile(list, m, i, true, false, false);
-                       if (!cmdout) {
-                               res = talloc(list, struct compile_tests_result);
-                               res->filename = i->name;
-                               res->description = "compiled successfully"
-                                       " with -DFAIL";
-                               res->output = "";
-                               list_add_tail(list, &res->list);
-                       }
-               }
-       }
-
-       if (list_empty(list)) {
-               talloc_free(list);
-               list = NULL;
-       }
-
-       return list;
-}
-
-static unsigned int score_compile_tests(struct manifest *m,
-                                       void *check_result)
-{
-       struct list_head *list = check_result;
-       struct compile_tests_result *i;
-       unsigned int score = compile_tests.total_score;
-
-       list_for_each(list, i, list)
-               score--;
-       return score;
-}
-
-static const char *describe_compile_tests(struct manifest *m,
-                                         void *check_result)
-{
-       struct list_head *list = check_result;
-       struct compile_tests_result *i;
-       char *descrip = talloc_strdup(list, "Compilation tests failed:\n");
-
-       list_for_each(list, i, list)
-               descrip = talloc_asprintf_append(descrip, "%s %s\n%s",
-                                                i->filename, i->description,
-                                                i->output);
-       return descrip;
-}
-
-struct ccanlint compile_tests = {
-       .key = "compile-tests",
-       .name = "Module tests compile",
-       .score = score_compile_tests,
-       .check = do_compile_tests,
-       .describe = describe_compile_tests,
-       .can_run = can_build,
-};
-
-REGISTER_TEST(compile_tests, &compile_test_helpers, &build_objs, NULL);
diff --git a/tools/ccanlint/compulsory_tests/has_tests.c b/tools/ccanlint/compulsory_tests/has_tests.c
deleted file mode 100644 (file)
index f6ef496..0000000
+++ /dev/null
@@ -1,137 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <ccan/talloc/talloc.h>
-
-static char test_is_not_dir[] = "test is not a directory";
-
-static void *check_has_tests(struct manifest *m,
-                            bool keep,
-                            unsigned int *timeleft)
-{
-       struct stat st;
-       char *test_dir = talloc_asprintf(m, "%s/test", m->dir);
-
-       if (lstat(test_dir, &st) != 0) {
-               if (errno != ENOENT)
-                       err(1, "statting %s", test_dir);
-               return "You have no test directory";
-       }
-
-       if (!S_ISDIR(st.st_mode))
-               return test_is_not_dir;
-
-       if (list_empty(&m->api_tests)
-           && list_empty(&m->run_tests)
-           && list_empty(&m->compile_ok_tests)) {
-               if (list_empty(&m->compile_fail_tests)) 
-                       return "You have no tests in the test directory";
-               else
-                       return "You have no positive tests in the test directory";
-       }
-       return NULL;
-}
-
-static const char *describe_has_tests(struct manifest *m, void *check_result)
-{
-       return talloc_asprintf(m, "%s\n\n"
-        "CCAN modules have a directory called test/ which contains tests.\n"
-       "There are four kinds of tests: api, run, compile_ok and compile_fail:\n"
-       "you can tell which type of test a C file is by its name, eg 'run.c'\n"
-       "and 'run-simple.c' are both run tests.\n\n"
-
-       "The simplest kind of test is a run test, which must compile with no\n"
-       "warnings, and then run: it is expected to use libtap to report its\n"
-       "results in a simple and portable format.  It should #include the C\n"
-       "files from the module directly (so it can probe the internals): the\n"
-       "module will not be linked in.  The test will be run in a temporary\n"
-       "directory, with the test directory symlinked under test/.\n\n"
-
-       "api tests are just like a run test, except it is a guarantee of API\n"
-       "stability: this test should pass on all future versions of the\n"
-       "module.  They *are* linked to the module, since they should only\n"
-       "test the API, not the internal state.\n\n"
-
-       "compile_ok tests are a subset of run tests: they must compile and\n"
-       "link, but aren't run.\n\n"
-
-       "compile_fail tests are tests which should fail to compile (or emit\n"
-       "warnings) or link when FAIL is defined, but should compile and link\n"
-       "when it's not defined: this helps ensure unrelated errors don't make\n"
-       "compilation fail.\n\n"
-
-       "Note that the tests are not linked against the files in the\n"
-       "above: you should directly #include those C files you want.  This\n"
-       "allows access to static functions and use special effects inside\n"
-       "test files\n", (char *)check_result);
-}
-
-static void handle_no_tests(struct manifest *m, void *check_result)
-{
-       FILE *run;
-       struct ccan_file *i;
-
-       if (check_result == test_is_not_dir)
-               return;
-
-       if (!ask("Should I create a template test/run.c file for you?"))
-               return;
-
-       if (mkdir("test", 0700) != 0) {
-               if (errno != EEXIST)
-                       err(1, "Creating test/ directory");
-       }
-
-       run = fopen("test/run.c", "w");
-       if (!run)
-               err(1, "Trying to create a test/run.c");
-
-       fputs("/* Include the main header first, to test it works */\n", run);
-       fprintf(run, "#include \"%s/%s.h\"\n", m->basename, m->basename);
-       fputs("/* Include the C files directly. */\n", run);
-       list_for_each(&m->c_files, i, list)
-               fprintf(run, "#include \"%s/%s\"\n", m->basename, i->name);
-       fputs("#include \"tap/tap.h\"\n", run);
-       fputs("\n", run);
-
-       fputs("int main(void)\n", run);
-       fputs("{\n", run);
-       fputs("\t/* This is how many tests you plan to run */\n", run);
-       fputs("\tplan_tests(3);\n", run);
-       fputs("\n", run);
-       fputs("\t/* Simple thing we expect to succeed */\n", run);
-       fputs("\tok1(some_test())\n", run);
-       fputs("\t/* Same, with an explicit description of the test. */\n", run);
-       fputs("\tok(some_test(), \"%s with no args should return 1\", \"some_test\")\n", run);
-       fputs("\t/* How to print out messages for debugging. */\n", run);
-       fputs("\tdiag(\"Address of some_test is %p\", &some_test)\n", run);
-       fputs("\t/* Conditional tests must be explicitly skipped. */\n", run);
-       fputs("#if HAVE_SOME_FEATURE\n", run);
-       fputs("\tok1(test_some_feature())\n", run);
-       fputs("#else\n", run);
-       fputs("\tskip(1, \"Don\'t have SOME_FEATURE\")\n", run);
-       fputs("#endif\n", run);
-       fputs("\n", run);
-       fputs("\t/* This exits depending on whether all tests passed */\n", run);
-       fputs("\treturn exit_status();\n", run);
-       fputs("}\n", run);
-
-       fclose(run);
-}      
-
-struct ccanlint has_tests = {
-       .key = "has-tests",
-       .name = "Module has tests",
-       .check = check_has_tests,
-       .describe = describe_has_tests,
-       .handle = handle_no_tests,
-};
-
-REGISTER_TEST(has_tests, NULL);
diff --git a/tools/ccanlint/compulsory_tests/run_tests.c b/tools/ccanlint/compulsory_tests/run_tests.c
deleted file mode 100644 (file)
index e2ab922..0000000
+++ /dev/null
@@ -1,124 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_run(struct manifest *m)
-{
-       if (safe_mode)
-               return "Safe mode enabled";
-       return NULL;
-}
-
-struct run_tests_result {
-       struct list_node list;
-       struct ccan_file *file;
-       const char *output;
-};
-
-static void *do_run_tests(struct manifest *m,
-                         bool keep,
-                         unsigned int *timeleft)
-{
-       struct list_head *list = talloc(m, struct list_head);
-       struct run_tests_result *res;
-       struct ccan_file *i;
-       char *cmdout;
-
-       list_head_init(list);
-
-       list_for_each(&m->run_tests, i, list) {
-               run_tests.total_score++;
-               cmdout = run_command(m, timeleft, i->compiled);
-               if (cmdout) {
-                       res = talloc(list, struct run_tests_result);
-                       res->file = i;
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
-       }
-
-       list_for_each(&m->api_tests, i, list) {
-               run_tests.total_score++;
-               cmdout = run_command(m, timeleft, i->compiled);
-               if (cmdout) {
-                       res = talloc(list, struct run_tests_result);
-                       res->file = i;
-                       res->output = talloc_steal(res, cmdout);
-                       list_add_tail(list, &res->list);
-               }
-       }
-
-       if (list_empty(list)) {
-               talloc_free(list);
-               list = NULL;
-       }
-
-       return list;
-}
-
-static unsigned int score_run_tests(struct manifest *m, void *check_result)
-{
-       struct list_head *list = check_result;
-       struct run_tests_result *i;
-       unsigned int score = run_tests.total_score;
-
-       list_for_each(list, i, list)
-               score--;
-       return score;
-}
-
-static const char *describe_run_tests(struct manifest *m,
-                                         void *check_result)
-{
-       struct list_head *list = check_result;
-       char *descrip = talloc_strdup(check_result, "Running tests failed:\n");
-       struct run_tests_result *i;
-
-       list_for_each(list, i, list)
-               descrip = talloc_asprintf_append(descrip, "Running %s:\n%s",
-                                                i->file->name, i->output);
-       return descrip;
-}
-
-/* Gcc's warn_unused_result is fascist bullshit. */
-#define doesnt_matter()
-
-static void run_under_debugger(struct manifest *m, void *check_result)
-{
-       char *command;
-       struct list_head *list = check_result;
-       struct run_tests_result *first;
-
-       if (!ask("Should I run the first failing test under the debugger?"))
-               return;
-
-       first = list_top(list, struct run_tests_result, list);
-       command = talloc_asprintf(m, "gdb -ex 'break tap.c:136' -ex 'run' %s",
-                                 first->file->compiled);
-       if (system(command))
-               doesnt_matter();
-}
-
-struct ccanlint run_tests = {
-       .key = "run",
-       .name = "Module's run and api tests pass",
-       .score = score_run_tests,
-       .check = do_run_tests,
-       .describe = describe_run_tests,
-       .can_run = can_run,
-       .handle = run_under_debugger
-};
-
-REGISTER_TEST(run_tests, &compile_tests, NULL);
index 7f75064219f495d514266559b5b25c9d79e94b91..98f7948d750b8af20aefce59d5a38c86093e3505 100644 (file)
@@ -190,6 +190,7 @@ struct ccanlint compile_coverage_tests = {
        .key = "compile-coverage-tests",
        .name = "Module tests compile with profiling",
        .check = do_compile_coverage_tests,
+       .total_score = 1,
        .describe = describe_compile_coverage_tests,
        .can_run = can_run_coverage,
 };
diff --git a/tools/ccanlint/tests/compile_test_helpers.c b/tools/ccanlint/tests/compile_test_helpers.c
new file mode 100644 (file)
index 0000000..0ab0cad
--- /dev/null
@@ -0,0 +1,68 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_build(struct manifest *m)
+{
+       if (safe_mode)
+               return "Safe mode enabled";
+       return NULL;
+}
+
+static char *compile(struct manifest *m,
+                    bool keep,
+                    struct ccan_file *cfile)
+{
+       cfile->compiled = maybe_temp_file(m, ".o", keep, cfile->fullname);
+       return compile_object(m, cfile->fullname, ccan_dir, "",
+                             cfile->compiled);
+}
+
+static void *do_compile_test_helpers(struct manifest *m,
+                                    bool keep,
+                                    unsigned int *timeleft)
+{
+       char *cmdout = NULL;
+       struct ccan_file *i;
+
+       compile_tests.total_score = 0;
+       list_for_each(&m->other_test_c_files, i, list) {
+               compile_tests.total_score++;
+               cmdout = compile(m, keep, i);
+               if (cmdout)
+                       return talloc_asprintf(m,
+                                              "Failed to compile helper C"
+                                              " code file %s:\n%s",
+                                              i->name, cmdout);
+       }
+       return NULL;
+}
+
+static const char *describe_compile_test_helpers(struct manifest *m,
+                                                void *check_result)
+{
+       return check_result;
+}
+
+struct ccanlint compile_test_helpers = {
+       .key = "compile-helpers",
+       .name = "Module test helper objects compile",
+       .total_score = 1,
+       .check = do_compile_test_helpers,
+       .describe = describe_compile_test_helpers,
+       .can_run = can_build,
+};
+
+REGISTER_TEST(compile_test_helpers, &depends_built, &has_tests, NULL);
diff --git a/tools/ccanlint/tests/compile_tests.c b/tools/ccanlint/tests/compile_tests.c
new file mode 100644 (file)
index 0000000..3d6d388
--- /dev/null
@@ -0,0 +1,206 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_build(struct manifest *m)
+{
+       if (safe_mode)
+               return "Safe mode enabled";
+       return NULL;
+}
+
+static char *obj_list(const struct manifest *m, bool link_with_module)
+{
+       char *list;
+       struct ccan_file *i;
+
+       /* We expect to be linked with tap, unless that's us. */
+       if (!streq(m->basename, "tap"))
+               list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir);
+       else
+               list = talloc_strdup(m, "");
+
+       /* Objects from any other C files. */
+       list_for_each(&m->other_test_c_files, i, list)
+               list = talloc_asprintf_append(list, " %s", i->compiled);
+
+       /* Our own object files. */
+       if (link_with_module)
+               list_for_each(&m->c_files, i, list)
+                       list = talloc_asprintf_append(list, " %s", i->compiled);
+
+       /* Other ccan modules. */
+       list_for_each(&m->dep_dirs, i, list) {
+               if (i->compiled)
+                       list = talloc_asprintf_append(list, " %s", i->compiled);
+       }
+
+       return list;
+}
+
+static char *lib_list(const struct manifest *m)
+{
+       unsigned int i, num;
+       char **libs = get_libs(m, ".", &num, &m->info_file->compiled);
+       char *ret = talloc_strdup(m, "");
+
+       for (i = 0; i < num; i++)
+               ret = talloc_asprintf_append(ret, "-l%s ", libs[i]);
+       return ret;
+}
+
+static char *compile(const void *ctx,
+                    struct manifest *m,
+                    struct ccan_file *file,
+                    bool fail,
+                    bool link_with_module,
+                    bool keep)
+{
+       char *errmsg;
+
+       file->compiled = maybe_temp_file(ctx, "", keep, file->fullname);
+       errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
+                                 obj_list(m, link_with_module),
+                                 fail ? "-DFAIL" : "",
+                                 lib_list(m), file->compiled);
+       if (errmsg) {
+               talloc_free(file->compiled);
+               return errmsg;
+       }
+       return NULL;
+}
+
+struct compile_tests_result {
+       struct list_node list;
+       const char *filename;
+       const char *description;
+       const char *output;
+};
+
+static void *do_compile_tests(struct manifest *m,
+                             bool keep,
+                             unsigned int *timeleft)
+{
+       struct list_head *list = talloc(m, struct list_head);
+       char *cmdout;
+       struct ccan_file *i;
+       struct compile_tests_result *res;
+
+       list_head_init(list);
+
+       compile_tests.total_score = 0;
+       list_for_each(&m->compile_ok_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = compile(list, m, i, false, false, keep);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       list_for_each(&m->run_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = compile(m, m, i, false, false, keep);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       list_for_each(&m->api_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = compile(m, m, i, false, true, keep);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       list_for_each(&m->compile_fail_tests, i, list) {
+               compile_tests.total_score++;
+               cmdout = compile(list, m, i, false, false, false);
+               if (cmdout) {
+                       res = talloc(list, struct compile_tests_result);
+                       res->filename = i->name;
+                       res->description = "failed to compile without -DFAIL";
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               } else {
+                       cmdout = compile(list, m, i, true, false, false);
+                       if (!cmdout) {
+                               res = talloc(list, struct compile_tests_result);
+                               res->filename = i->name;
+                               res->description = "compiled successfully"
+                                       " with -DFAIL";
+                               res->output = "";
+                               list_add_tail(list, &res->list);
+                       }
+               }
+       }
+
+       if (list_empty(list)) {
+               talloc_free(list);
+               list = NULL;
+       }
+
+       return list;
+}
+
+static unsigned int score_compile_tests(struct manifest *m,
+                                       void *check_result)
+{
+       struct list_head *list = check_result;
+       struct compile_tests_result *i;
+       unsigned int score = compile_tests.total_score;
+
+       list_for_each(list, i, list)
+               score--;
+       return score;
+}
+
+static const char *describe_compile_tests(struct manifest *m,
+                                         void *check_result)
+{
+       struct list_head *list = check_result;
+       struct compile_tests_result *i;
+       char *descrip = talloc_strdup(list, "Compilation tests failed:\n");
+
+       list_for_each(list, i, list)
+               descrip = talloc_asprintf_append(descrip, "%s %s\n%s",
+                                                i->filename, i->description,
+                                                i->output);
+       return descrip;
+}
+
+struct ccanlint compile_tests = {
+       .key = "compile-tests",
+       .name = "Module tests compile",
+       .score = score_compile_tests,
+       .total_score = 1,
+       .check = do_compile_tests,
+       .describe = describe_compile_tests,
+       .can_run = can_build,
+};
+
+REGISTER_TEST(compile_tests, &compile_test_helpers, &build_objs, NULL);
index 82b479620c614ba6c1e7d995003b60bdd96a0864..e66379f5e7fee45e9036a5cda09f664dac568b09 100644 (file)
@@ -506,6 +506,11 @@ static void *build_examples(struct manifest *m, bool keep,
                /* This didn't work, so not a candidate for combining. */
                prev = NULL;
        }
+
+       if (strcmp(score->errors, "") == 0) {
+               talloc_free(score);
+               return NULL;
+       }
        return score;
 }
 
diff --git a/tools/ccanlint/tests/has_tests.c b/tools/ccanlint/tests/has_tests.c
new file mode 100644 (file)
index 0000000..56eaf2a
--- /dev/null
@@ -0,0 +1,138 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <ccan/talloc/talloc.h>
+
+static char test_is_not_dir[] = "test is not a directory";
+
+static void *check_has_tests(struct manifest *m,
+                            bool keep,
+                            unsigned int *timeleft)
+{
+       struct stat st;
+       char *test_dir = talloc_asprintf(m, "%s/test", m->dir);
+
+       if (lstat(test_dir, &st) != 0) {
+               if (errno != ENOENT)
+                       err(1, "statting %s", test_dir);
+               return "You have no test directory";
+       }
+
+       if (!S_ISDIR(st.st_mode))
+               return test_is_not_dir;
+
+       if (list_empty(&m->api_tests)
+           && list_empty(&m->run_tests)
+           && list_empty(&m->compile_ok_tests)) {
+               if (list_empty(&m->compile_fail_tests))
+                       return "You have no tests in the test directory";
+               else
+                       return "You have no positive tests in the test directory";
+       }
+       return NULL;
+}
+
+static const char *describe_has_tests(struct manifest *m, void *check_result)
+{
+       return talloc_asprintf(m, "%s\n\n"
+        "CCAN modules have a directory called test/ which contains tests.\n"
+       "There are four kinds of tests: api, run, compile_ok and compile_fail:\n"
+       "you can tell which type of test a C file is by its name, eg 'run.c'\n"
+       "and 'run-simple.c' are both run tests.\n\n"
+
+       "The simplest kind of test is a run test, which must compile with no\n"
+       "warnings, and then run: it is expected to use libtap to report its\n"
+       "results in a simple and portable format.  It should #include the C\n"
+       "files from the module directly (so it can probe the internals): the\n"
+       "module will not be linked in.  The test will be run in a temporary\n"
+       "directory, with the test directory symlinked under test/.\n\n"
+
+       "api tests are just like a run test, except it is a guarantee of API\n"
+       "stability: this test should pass on all future versions of the\n"
+       "module.  They *are* linked to the module, since they should only\n"
+       "test the API, not the internal state.\n\n"
+
+       "compile_ok tests are a subset of run tests: they must compile and\n"
+       "link, but aren't run.\n\n"
+
+       "compile_fail tests are tests which should fail to compile (or emit\n"
+       "warnings) or link when FAIL is defined, but should compile and link\n"
+       "when it's not defined: this helps ensure unrelated errors don't make\n"
+       "compilation fail.\n\n"
+
+       "Note that the tests are not linked against the files in the\n"
+       "above: you should directly #include those C files you want.  This\n"
+       "allows access to static functions and use special effects inside\n"
+       "test files\n", (char *)check_result);
+}
+
+static void handle_no_tests(struct manifest *m, void *check_result)
+{
+       FILE *run;
+       struct ccan_file *i;
+
+       if (check_result == test_is_not_dir)
+               return;
+
+       if (!ask("Should I create a template test/run.c file for you?"))
+               return;
+
+       if (mkdir("test", 0700) != 0) {
+               if (errno != EEXIST)
+                       err(1, "Creating test/ directory");
+       }
+
+       run = fopen("test/run.c", "w");
+       if (!run)
+               err(1, "Trying to create a test/run.c");
+
+       fputs("/* Include the main header first, to test it works */\n", run);
+       fprintf(run, "#include \"%s/%s.h\"\n", m->basename, m->basename);
+       fputs("/* Include the C files directly. */\n", run);
+       list_for_each(&m->c_files, i, list)
+               fprintf(run, "#include \"%s/%s\"\n", m->basename, i->name);
+       fputs("#include \"tap/tap.h\"\n", run);
+       fputs("\n", run);
+
+       fputs("int main(void)\n", run);
+       fputs("{\n", run);
+       fputs("\t/* This is how many tests you plan to run */\n", run);
+       fputs("\tplan_tests(3);\n", run);
+       fputs("\n", run);
+       fputs("\t/* Simple thing we expect to succeed */\n", run);
+       fputs("\tok1(some_test())\n", run);
+       fputs("\t/* Same, with an explicit description of the test. */\n", run);
+       fputs("\tok(some_test(), \"%s with no args should return 1\", \"some_test\")\n", run);
+       fputs("\t/* How to print out messages for debugging. */\n", run);
+       fputs("\tdiag(\"Address of some_test is %p\", &some_test)\n", run);
+       fputs("\t/* Conditional tests must be explicitly skipped. */\n", run);
+       fputs("#if HAVE_SOME_FEATURE\n", run);
+       fputs("\tok1(test_some_feature())\n", run);
+       fputs("#else\n", run);
+       fputs("\tskip(1, \"Don\'t have SOME_FEATURE\")\n", run);
+       fputs("#endif\n", run);
+       fputs("\n", run);
+       fputs("\t/* This exits depending on whether all tests passed */\n", run);
+       fputs("\treturn exit_status();\n", run);
+       fputs("}\n", run);
+
+       fclose(run);
+}
+
+struct ccanlint has_tests = {
+       .key = "has-tests",
+       .name = "Module has tests",
+       .check = check_has_tests,
+       .describe = describe_has_tests,
+       .total_score = 1,
+       .handle = handle_no_tests,
+};
+
+REGISTER_TEST(has_tests, NULL);
index 886aebd54424b9019eb212eca4a91e649c799dd7..a1bd8dea59f79a4a145d16594408182f66fe01b4 100644 (file)
@@ -209,4 +209,4 @@ struct ccanlint run_coverage_tests = {
        .describe = describe_run_coverage_tests,
 };
 
-REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);
+REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, &run_tests, NULL);
diff --git a/tools/ccanlint/tests/run_tests.c b/tools/ccanlint/tests/run_tests.c
new file mode 100644 (file)
index 0000000..559a637
--- /dev/null
@@ -0,0 +1,126 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_run(struct manifest *m)
+{
+       if (safe_mode)
+               return "Safe mode enabled";
+       return NULL;
+}
+
+struct run_tests_result {
+       struct list_node list;
+       struct ccan_file *file;
+       const char *output;
+};
+
+static void *do_run_tests(struct manifest *m,
+                         bool keep,
+                         unsigned int *timeleft)
+{
+       struct list_head *list = talloc(m, struct list_head);
+       struct run_tests_result *res;
+       struct ccan_file *i;
+       char *cmdout;
+
+       list_head_init(list);
+       run_tests.total_score = 0;
+
+       list_for_each(&m->run_tests, i, list) {
+               run_tests.total_score++;
+               cmdout = run_command(m, timeleft, i->compiled);
+               if (cmdout) {
+                       res = talloc(list, struct run_tests_result);
+                       res->file = i;
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       list_for_each(&m->api_tests, i, list) {
+               run_tests.total_score++;
+               cmdout = run_command(m, timeleft, i->compiled);
+               if (cmdout) {
+                       res = talloc(list, struct run_tests_result);
+                       res->file = i;
+                       res->output = talloc_steal(res, cmdout);
+                       list_add_tail(list, &res->list);
+               }
+       }
+
+       if (list_empty(list)) {
+               talloc_free(list);
+               list = NULL;
+       }
+
+       return list;
+}
+
+static unsigned int score_run_tests(struct manifest *m, void *check_result)
+{
+       struct list_head *list = check_result;
+       struct run_tests_result *i;
+       unsigned int score = run_tests.total_score;
+
+       list_for_each(list, i, list)
+               score--;
+       return score;
+}
+
+static const char *describe_run_tests(struct manifest *m,
+                                         void *check_result)
+{
+       struct list_head *list = check_result;
+       char *descrip = talloc_strdup(check_result, "Running tests failed:\n");
+       struct run_tests_result *i;
+
+       list_for_each(list, i, list)
+               descrip = talloc_asprintf_append(descrip, "Running %s:\n%s",
+                                                i->file->name, i->output);
+       return descrip;
+}
+
+/* Gcc's warn_unused_result is fascist bullshit. */
+#define doesnt_matter()
+
+static void run_under_debugger(struct manifest *m, void *check_result)
+{
+       char *command;
+       struct list_head *list = check_result;
+       struct run_tests_result *first;
+
+       if (!ask("Should I run the first failing test under the debugger?"))
+               return;
+
+       first = list_top(list, struct run_tests_result, list);
+       command = talloc_asprintf(m, "gdb -ex 'break tap.c:136' -ex 'run' %s",
+                                 first->file->compiled);
+       if (system(command))
+               doesnt_matter();
+}
+
+struct ccanlint run_tests = {
+       .key = "run",
+       .name = "Module's run and api tests pass",
+       .score = score_run_tests,
+       .total_score = 1,
+       .check = do_run_tests,
+       .describe = describe_run_tests,
+       .can_run = can_run,
+       .handle = run_under_debugger
+};
+
+REGISTER_TEST(run_tests, &compile_tests, NULL);
index 4b425ea23ba59d7baa3d689ddbbd58355596af8f..caf8323c2cfa26cf53257d226f698e69ac6a9a6c 100644 (file)
@@ -41,6 +41,7 @@ static void *do_run_tests_vg(struct manifest *m,
        char *cmdout;
 
        list_head_init(list);
+       run_tests_vg.total_score = 0;
 
        list_for_each(&m->run_tests, i, list) {
                run_tests_vg.total_score++;
@@ -123,6 +124,7 @@ struct ccanlint run_tests_vg = {
        .key = "valgrind-tests",
        .name = "Module's run and api tests succeed under valgrind",
        .score = score_run_tests_vg,
+       .total_score = 1,
        .check = do_run_tests_vg,
        .describe = describe_run_tests_vg,
        .can_run = can_run_vg,