From: Rusty Russell <rusty@rustcorp.com.au>
Date: Wed, 3 Nov 2010 12:55:57 +0000 (+1030)
Subject: ccanlint: make tests non-compulsory, always print score.
X-Git-Url: http://git.ozlabs.org/?a=commitdiff_plain;h=374d3d2eaa1b29dd3f3a940ca472bb209e95155d;p=ccan

ccanlint: make tests non-compulsory, always print score.

No longer abort ccanlint because a test fails, but you will get a low
score if there are no tests.

(Note: total_score is filled in, even if it's overridden when the test is
 run.  This means that if the prereq fails, that total counts so there is
 still a penalty!)
---

diff --git a/tools/ccanlint/ccanlint.c b/tools/ccanlint/ccanlint.c
index ab805806..01a94758 100644
--- a/tools/ccanlint/ccanlint.c
+++ b/tools/ccanlint/ccanlint.c
@@ -407,12 +407,13 @@ int main(int argc, char *argv[])
 		    talloc_asprintf(m, "%s/test", temp_dir(NULL))) != 0)
 		err(1, "Creating test symlink in %s", temp_dir(NULL));
 
-	/* If you don't pass the compulsory tests, you don't even get a score */
+	/* If you don't pass the compulsory tests, you get a score of 0. */
 	if (verbose)
 		printf("Compulsory tests:\n");
 
 	while ((i = get_next_test(&compulsory_tests)) != NULL) {
 		if (!run_test(i, summary, &score, &total_score, m)) {
+			printf("%sTotal score: 0/%u\n", prefix, total_score);
 			errx(1, "%s%s failed", prefix, i->name);
 		}
 	}
diff --git a/tools/ccanlint/compulsory_tests/compile_test_helpers.c b/tools/ccanlint/compulsory_tests/compile_test_helpers.c
deleted file mode 100644
index 207c0b2d..00000000
--- a/tools/ccanlint/compulsory_tests/compile_test_helpers.c
+++ /dev/null
@@ -1,67 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_build(struct manifest *m)
-{
-	if (safe_mode)
-		return "Safe mode enabled";
-	return NULL;
-}
-
-static char *compile(struct manifest *m, 
-		     bool keep,
-		     struct ccan_file *cfile)
-{
-	cfile->compiled = maybe_temp_file(m, ".o", keep, cfile->fullname);
-	return compile_object(m, cfile->fullname, ccan_dir, "",
-			      cfile->compiled);
-}
-
-static void *do_compile_test_helpers(struct manifest *m,
-				     bool keep,
-				     unsigned int *timeleft)
-{
-	char *cmdout = NULL;
-	struct ccan_file *i;
-
-	list_for_each(&m->other_test_c_files, i, list) {
-		compile_tests.total_score++;
-		cmdout = compile(m, keep, i);
-		if (cmdout)
-			return talloc_asprintf(m,
-					       "Failed to compile helper C"
-					       " code file %s:\n%s",
-					       i->name, cmdout);
-	}
-	return NULL;
-}
-
-static const char *describe_compile_test_helpers(struct manifest *m,
-						 void *check_result)
-{
-	return check_result;
-}
-
-struct ccanlint compile_test_helpers = {
-	.key = "compile-helpers",
-	.name = "Module test helper objects compile",
-	.total_score = 1,
-	.check = do_compile_test_helpers,
-	.describe = describe_compile_test_helpers,
-	.can_run = can_build,
-};
-
-REGISTER_TEST(compile_test_helpers, &depends_built);
diff --git a/tools/ccanlint/compulsory_tests/compile_tests.c b/tools/ccanlint/compulsory_tests/compile_tests.c
deleted file mode 100644
index 1559bd18..00000000
--- a/tools/ccanlint/compulsory_tests/compile_tests.c
+++ /dev/null
@@ -1,204 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_build(struct manifest *m)
-{
-	if (safe_mode)
-		return "Safe mode enabled";
-	return NULL;
-}
-
-static char *obj_list(const struct manifest *m, bool link_with_module)
-{
-	char *list;
-	struct ccan_file *i;
-
-	/* We expect to be linked with tap, unless that's us. */
-	if (!streq(m->basename, "tap"))
-		list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir);
-	else
-		list = talloc_strdup(m, "");
-
-	/* Objects from any other C files. */
-	list_for_each(&m->other_test_c_files, i, list)
-		list = talloc_asprintf_append(list, " %s", i->compiled);
-
-	/* Our own object files. */
-	if (link_with_module)
-		list_for_each(&m->c_files, i, list)
-			list = talloc_asprintf_append(list, " %s", i->compiled);
-
-	/* Other ccan modules. */
-	list_for_each(&m->dep_dirs, i, list) {
-		if (i->compiled)
-			list = talloc_asprintf_append(list, " %s", i->compiled);
-	}
-
-	return list;
-}
-
-static char *lib_list(const struct manifest *m)
-{
-	unsigned int i, num;
-	char **libs = get_libs(m, ".", &num, &m->info_file->compiled);
-	char *ret = talloc_strdup(m, "");
-
-	for (i = 0; i < num; i++)
-		ret = talloc_asprintf_append(ret, "-l%s ", libs[i]);
-	return ret;
-}
-
-static char *compile(const void *ctx,
-		     struct manifest *m,
-		     struct ccan_file *file,
-		     bool fail,
-		     bool link_with_module,
-		     bool keep)
-{
-	char *errmsg;
-
-	file->compiled = maybe_temp_file(ctx, "", keep, file->fullname);
-	errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
-				  obj_list(m, link_with_module),
-				  fail ? "-DFAIL" : "",
-				  lib_list(m), file->compiled);
-	if (errmsg) {
-		talloc_free(file->compiled);
-		return errmsg;
-	}
-	return NULL;
-}
-
-struct compile_tests_result {
-	struct list_node list;
-	const char *filename;
-	const char *description;
-	const char *output;
-};
-
-static void *do_compile_tests(struct manifest *m,
-			      bool keep,
-			      unsigned int *timeleft)
-{
-	struct list_head *list = talloc(m, struct list_head);
-	char *cmdout;
-	struct ccan_file *i;
-	struct compile_tests_result *res;
-
-	list_head_init(list);
-
-	list_for_each(&m->compile_ok_tests, i, list) {
-		compile_tests.total_score++;
-		cmdout = compile(list, m, i, false, false, keep);
-		if (cmdout) {
-			res = talloc(list, struct compile_tests_result);
-			res->filename = i->name;
-			res->description = "failed to compile";
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		}
-	}
-
-	list_for_each(&m->run_tests, i, list) {
-		compile_tests.total_score++;
-		cmdout = compile(m, m, i, false, false, keep);
-		if (cmdout) {
-			res = talloc(list, struct compile_tests_result);
-			res->filename = i->name;
-			res->description = "failed to compile";
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		}
-	}
-
-	list_for_each(&m->api_tests, i, list) {
-		compile_tests.total_score++;
-		cmdout = compile(m, m, i, false, true, keep);
-		if (cmdout) {
-			res = talloc(list, struct compile_tests_result);
-			res->filename = i->name;
-			res->description = "failed to compile";
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		}
-	}
-
-	list_for_each(&m->compile_fail_tests, i, list) {
-		compile_tests.total_score++;
-		cmdout = compile(list, m, i, false, false, false);
-		if (cmdout) {
-			res = talloc(list, struct compile_tests_result);
-			res->filename = i->name;
-			res->description = "failed to compile without -DFAIL";
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		} else {
-			cmdout = compile(list, m, i, true, false, false);
-			if (!cmdout) {
-				res = talloc(list, struct compile_tests_result);
-				res->filename = i->name;
-				res->description = "compiled successfully"
-					" with -DFAIL";
-				res->output = "";
-				list_add_tail(list, &res->list);
-			}
-		}
-	}
-
-	if (list_empty(list)) {
-		talloc_free(list);
-		list = NULL;
-	}
-
-	return list;
-}
-
-static unsigned int score_compile_tests(struct manifest *m,
-					void *check_result)
-{
-	struct list_head *list = check_result;
-	struct compile_tests_result *i;
-	unsigned int score = compile_tests.total_score;
-
-	list_for_each(list, i, list)
-		score--;
-	return score;
-}
-
-static const char *describe_compile_tests(struct manifest *m,
-					  void *check_result)
-{
-	struct list_head *list = check_result;
-	struct compile_tests_result *i;
-	char *descrip = talloc_strdup(list, "Compilation tests failed:\n");
-
-	list_for_each(list, i, list)
-		descrip = talloc_asprintf_append(descrip, "%s %s\n%s",
-						 i->filename, i->description,
-						 i->output);
-	return descrip;
-}
-
-struct ccanlint compile_tests = {
-	.key = "compile-tests",
-	.name = "Module tests compile",
-	.score = score_compile_tests,
-	.check = do_compile_tests,
-	.describe = describe_compile_tests,
-	.can_run = can_build,
-};
-
-REGISTER_TEST(compile_tests, &compile_test_helpers, &build_objs, NULL);
diff --git a/tools/ccanlint/compulsory_tests/has_tests.c b/tools/ccanlint/compulsory_tests/has_tests.c
deleted file mode 100644
index f6ef4968..00000000
--- a/tools/ccanlint/compulsory_tests/has_tests.c
+++ /dev/null
@@ -1,137 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <ccan/talloc/talloc.h>
-
-static char test_is_not_dir[] = "test is not a directory";
-
-static void *check_has_tests(struct manifest *m,
-			     bool keep,
-			     unsigned int *timeleft)
-{
-	struct stat st;
-	char *test_dir = talloc_asprintf(m, "%s/test", m->dir);
-
-	if (lstat(test_dir, &st) != 0) {
-		if (errno != ENOENT)
-			err(1, "statting %s", test_dir);
-		return "You have no test directory";
-	}
-
-	if (!S_ISDIR(st.st_mode))
-		return test_is_not_dir;
-
-	if (list_empty(&m->api_tests)
-	    && list_empty(&m->run_tests)
-	    && list_empty(&m->compile_ok_tests)) {
-		if (list_empty(&m->compile_fail_tests)) 
-			return "You have no tests in the test directory";
-		else
-			return "You have no positive tests in the test directory";
-	}
-	return NULL;
-}
-
-static const char *describe_has_tests(struct manifest *m, void *check_result)
-{
-	return talloc_asprintf(m, "%s\n\n"
-        "CCAN modules have a directory called test/ which contains tests.\n"
-	"There are four kinds of tests: api, run, compile_ok and compile_fail:\n"
-	"you can tell which type of test a C file is by its name, eg 'run.c'\n"
-	"and 'run-simple.c' are both run tests.\n\n"
-
-	"The simplest kind of test is a run test, which must compile with no\n"
-	"warnings, and then run: it is expected to use libtap to report its\n"
-	"results in a simple and portable format.  It should #include the C\n"
-	"files from the module directly (so it can probe the internals): the\n"
-	"module will not be linked in.  The test will be run in a temporary\n"
-	"directory, with the test directory symlinked under test/.\n\n"
-
-	"api tests are just like a run test, except it is a guarantee of API\n"
-	"stability: this test should pass on all future versions of the\n"
-	"module.  They *are* linked to the module, since they should only\n"
-	"test the API, not the internal state.\n\n"
-
-	"compile_ok tests are a subset of run tests: they must compile and\n"
-	"link, but aren't run.\n\n"
-
-	"compile_fail tests are tests which should fail to compile (or emit\n"
-	"warnings) or link when FAIL is defined, but should compile and link\n"
-	"when it's not defined: this helps ensure unrelated errors don't make\n"
-	"compilation fail.\n\n"
-
-	"Note that the tests are not linked against the files in the\n"
-	"above: you should directly #include those C files you want.  This\n"
-	"allows access to static functions and use special effects inside\n"
-	"test files\n", (char *)check_result);
-}
-
-static void handle_no_tests(struct manifest *m, void *check_result)
-{
-	FILE *run;
-	struct ccan_file *i;
-
-	if (check_result == test_is_not_dir)
-		return;
-
-	if (!ask("Should I create a template test/run.c file for you?"))
-		return;
-
-	if (mkdir("test", 0700) != 0) {
-		if (errno != EEXIST)
-			err(1, "Creating test/ directory");
-	}
-
-	run = fopen("test/run.c", "w");
-	if (!run)
-		err(1, "Trying to create a test/run.c");
-
-	fputs("/* Include the main header first, to test it works */\n", run);
-	fprintf(run, "#include \"%s/%s.h\"\n", m->basename, m->basename);
-	fputs("/* Include the C files directly. */\n", run);
-	list_for_each(&m->c_files, i, list)
-		fprintf(run, "#include \"%s/%s\"\n", m->basename, i->name);
-	fputs("#include \"tap/tap.h\"\n", run);
-	fputs("\n", run);
-
-	fputs("int main(void)\n", run);
-	fputs("{\n", run);
-	fputs("\t/* This is how many tests you plan to run */\n", run);
-	fputs("\tplan_tests(3);\n", run);
-	fputs("\n", run);
-	fputs("\t/* Simple thing we expect to succeed */\n", run);
-	fputs("\tok1(some_test())\n", run);
-	fputs("\t/* Same, with an explicit description of the test. */\n", run);
-	fputs("\tok(some_test(), \"%s with no args should return 1\", \"some_test\")\n", run);
-	fputs("\t/* How to print out messages for debugging. */\n", run);
-	fputs("\tdiag(\"Address of some_test is %p\", &some_test)\n", run);
-	fputs("\t/* Conditional tests must be explicitly skipped. */\n", run);
-	fputs("#if HAVE_SOME_FEATURE\n", run);
-	fputs("\tok1(test_some_feature())\n", run);
-	fputs("#else\n", run);
-	fputs("\tskip(1, \"Don\'t have SOME_FEATURE\")\n", run);
-	fputs("#endif\n", run);
-	fputs("\n", run);
-	fputs("\t/* This exits depending on whether all tests passed */\n", run);
-	fputs("\treturn exit_status();\n", run);
-	fputs("}\n", run);
-
-	fclose(run);
-}	
-
-struct ccanlint has_tests = {
-	.key = "has-tests",
-	.name = "Module has tests",
-	.check = check_has_tests,
-	.describe = describe_has_tests,
-	.handle = handle_no_tests,
-};
-
-REGISTER_TEST(has_tests, NULL);
diff --git a/tools/ccanlint/compulsory_tests/run_tests.c b/tools/ccanlint/compulsory_tests/run_tests.c
deleted file mode 100644
index e2ab9226..00000000
--- a/tools/ccanlint/compulsory_tests/run_tests.c
+++ /dev/null
@@ -1,124 +0,0 @@
-#include <tools/ccanlint/ccanlint.h>
-#include <tools/tools.h>
-#include <ccan/talloc/talloc.h>
-#include <ccan/str/str.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <limits.h>
-#include <errno.h>
-#include <stdlib.h>
-#include <stdio.h>
-#include <err.h>
-#include <string.h>
-#include <ctype.h>
-
-static const char *can_run(struct manifest *m)
-{
-	if (safe_mode)
-		return "Safe mode enabled";
-	return NULL;
-}
-
-struct run_tests_result {
-	struct list_node list;
-	struct ccan_file *file;
-	const char *output;
-};
-
-static void *do_run_tests(struct manifest *m,
-			  bool keep,
-			  unsigned int *timeleft)
-{
-	struct list_head *list = talloc(m, struct list_head);
-	struct run_tests_result *res;
-	struct ccan_file *i;
-	char *cmdout;
-
-	list_head_init(list);
-
-	list_for_each(&m->run_tests, i, list) {
-		run_tests.total_score++;
-		cmdout = run_command(m, timeleft, i->compiled);
-		if (cmdout) {
-			res = talloc(list, struct run_tests_result);
-			res->file = i;
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		}
-	}
-
-	list_for_each(&m->api_tests, i, list) {
-		run_tests.total_score++;
-		cmdout = run_command(m, timeleft, i->compiled);
-		if (cmdout) {
-			res = talloc(list, struct run_tests_result);
-			res->file = i;
-			res->output = talloc_steal(res, cmdout);
-			list_add_tail(list, &res->list);
-		}
-	}
-
-	if (list_empty(list)) {
-		talloc_free(list);
-		list = NULL;
-	}
-
-	return list;
-}
-
-static unsigned int score_run_tests(struct manifest *m, void *check_result)
-{
-	struct list_head *list = check_result;
-	struct run_tests_result *i;
-	unsigned int score = run_tests.total_score;
-
-	list_for_each(list, i, list)
-		score--;
-	return score;
-}
-
-static const char *describe_run_tests(struct manifest *m,
-					  void *check_result)
-{
-	struct list_head *list = check_result;
-	char *descrip = talloc_strdup(check_result, "Running tests failed:\n");
-	struct run_tests_result *i;
-
-	list_for_each(list, i, list)
-		descrip = talloc_asprintf_append(descrip, "Running %s:\n%s",
-						 i->file->name, i->output);
-	return descrip;
-}
-
-/* Gcc's warn_unused_result is fascist bullshit. */
-#define doesnt_matter()
-
-static void run_under_debugger(struct manifest *m, void *check_result)
-{
-	char *command;
-	struct list_head *list = check_result;
-	struct run_tests_result *first;
-
-	if (!ask("Should I run the first failing test under the debugger?"))
-		return;
-
-	first = list_top(list, struct run_tests_result, list);
-	command = talloc_asprintf(m, "gdb -ex 'break tap.c:136' -ex 'run' %s",
-				  first->file->compiled);
-	if (system(command))
-		doesnt_matter();
-}
-
-struct ccanlint run_tests = {
-	.key = "run",
-	.name = "Module's run and api tests pass",
-	.score = score_run_tests,
-	.check = do_run_tests,
-	.describe = describe_run_tests,
-	.can_run = can_run,
-	.handle = run_under_debugger
-};
-
-REGISTER_TEST(run_tests, &compile_tests, NULL);
diff --git a/tools/ccanlint/tests/build-coverage.c b/tools/ccanlint/tests/build-coverage.c
index 7f750642..98f7948d 100644
--- a/tools/ccanlint/tests/build-coverage.c
+++ b/tools/ccanlint/tests/build-coverage.c
@@ -190,6 +190,7 @@ struct ccanlint compile_coverage_tests = {
 	.key = "compile-coverage-tests",
 	.name = "Module tests compile with profiling",
 	.check = do_compile_coverage_tests,
+	.total_score = 1,
 	.describe = describe_compile_coverage_tests,
 	.can_run = can_run_coverage,
 };
diff --git a/tools/ccanlint/tests/compile_test_helpers.c b/tools/ccanlint/tests/compile_test_helpers.c
new file mode 100644
index 00000000..0ab0cade
--- /dev/null
+++ b/tools/ccanlint/tests/compile_test_helpers.c
@@ -0,0 +1,68 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_build(struct manifest *m)
+{
+	if (safe_mode)
+		return "Safe mode enabled";
+	return NULL;
+}
+
+static char *compile(struct manifest *m,
+		     bool keep,
+		     struct ccan_file *cfile)
+{
+	cfile->compiled = maybe_temp_file(m, ".o", keep, cfile->fullname);
+	return compile_object(m, cfile->fullname, ccan_dir, "",
+			      cfile->compiled);
+}
+
+static void *do_compile_test_helpers(struct manifest *m,
+				     bool keep,
+				     unsigned int *timeleft)
+{
+	char *cmdout = NULL;
+	struct ccan_file *i;
+
+	compile_tests.total_score = 0;
+	list_for_each(&m->other_test_c_files, i, list) {
+		compile_tests.total_score++;
+		cmdout = compile(m, keep, i);
+		if (cmdout)
+			return talloc_asprintf(m,
+					       "Failed to compile helper C"
+					       " code file %s:\n%s",
+					       i->name, cmdout);
+	}
+	return NULL;
+}
+
+static const char *describe_compile_test_helpers(struct manifest *m,
+						 void *check_result)
+{
+	return check_result;
+}
+
+struct ccanlint compile_test_helpers = {
+	.key = "compile-helpers",
+	.name = "Module test helper objects compile",
+	.total_score = 1,
+	.check = do_compile_test_helpers,
+	.describe = describe_compile_test_helpers,
+	.can_run = can_build,
+};
+
+REGISTER_TEST(compile_test_helpers, &depends_built, &has_tests, NULL);
diff --git a/tools/ccanlint/tests/compile_tests.c b/tools/ccanlint/tests/compile_tests.c
new file mode 100644
index 00000000..3d6d3881
--- /dev/null
+++ b/tools/ccanlint/tests/compile_tests.c
@@ -0,0 +1,206 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_build(struct manifest *m)
+{
+	if (safe_mode)
+		return "Safe mode enabled";
+	return NULL;
+}
+
+static char *obj_list(const struct manifest *m, bool link_with_module)
+{
+	char *list;
+	struct ccan_file *i;
+
+	/* We expect to be linked with tap, unless that's us. */
+	if (!streq(m->basename, "tap"))
+		list = talloc_asprintf(m, "%s/ccan/tap.o", ccan_dir);
+	else
+		list = talloc_strdup(m, "");
+
+	/* Objects from any other C files. */
+	list_for_each(&m->other_test_c_files, i, list)
+		list = talloc_asprintf_append(list, " %s", i->compiled);
+
+	/* Our own object files. */
+	if (link_with_module)
+		list_for_each(&m->c_files, i, list)
+			list = talloc_asprintf_append(list, " %s", i->compiled);
+
+	/* Other ccan modules. */
+	list_for_each(&m->dep_dirs, i, list) {
+		if (i->compiled)
+			list = talloc_asprintf_append(list, " %s", i->compiled);
+	}
+
+	return list;
+}
+
+static char *lib_list(const struct manifest *m)
+{
+	unsigned int i, num;
+	char **libs = get_libs(m, ".", &num, &m->info_file->compiled);
+	char *ret = talloc_strdup(m, "");
+
+	for (i = 0; i < num; i++)
+		ret = talloc_asprintf_append(ret, "-l%s ", libs[i]);
+	return ret;
+}
+
+static char *compile(const void *ctx,
+		     struct manifest *m,
+		     struct ccan_file *file,
+		     bool fail,
+		     bool link_with_module,
+		     bool keep)
+{
+	char *errmsg;
+
+	file->compiled = maybe_temp_file(ctx, "", keep, file->fullname);
+	errmsg = compile_and_link(ctx, file->fullname, ccan_dir,
+				  obj_list(m, link_with_module),
+				  fail ? "-DFAIL" : "",
+				  lib_list(m), file->compiled);
+	if (errmsg) {
+		talloc_free(file->compiled);
+		return errmsg;
+	}
+	return NULL;
+}
+
+struct compile_tests_result {
+	struct list_node list;
+	const char *filename;
+	const char *description;
+	const char *output;
+};
+
+static void *do_compile_tests(struct manifest *m,
+			      bool keep,
+			      unsigned int *timeleft)
+{
+	struct list_head *list = talloc(m, struct list_head);
+	char *cmdout;
+	struct ccan_file *i;
+	struct compile_tests_result *res;
+
+	list_head_init(list);
+
+	compile_tests.total_score = 0;
+	list_for_each(&m->compile_ok_tests, i, list) {
+		compile_tests.total_score++;
+		cmdout = compile(list, m, i, false, false, keep);
+		if (cmdout) {
+			res = talloc(list, struct compile_tests_result);
+			res->filename = i->name;
+			res->description = "failed to compile";
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		}
+	}
+
+	list_for_each(&m->run_tests, i, list) {
+		compile_tests.total_score++;
+		cmdout = compile(m, m, i, false, false, keep);
+		if (cmdout) {
+			res = talloc(list, struct compile_tests_result);
+			res->filename = i->name;
+			res->description = "failed to compile";
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		}
+	}
+
+	list_for_each(&m->api_tests, i, list) {
+		compile_tests.total_score++;
+		cmdout = compile(m, m, i, false, true, keep);
+		if (cmdout) {
+			res = talloc(list, struct compile_tests_result);
+			res->filename = i->name;
+			res->description = "failed to compile";
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		}
+	}
+
+	list_for_each(&m->compile_fail_tests, i, list) {
+		compile_tests.total_score++;
+		cmdout = compile(list, m, i, false, false, false);
+		if (cmdout) {
+			res = talloc(list, struct compile_tests_result);
+			res->filename = i->name;
+			res->description = "failed to compile without -DFAIL";
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		} else {
+			cmdout = compile(list, m, i, true, false, false);
+			if (!cmdout) {
+				res = talloc(list, struct compile_tests_result);
+				res->filename = i->name;
+				res->description = "compiled successfully"
+					" with -DFAIL";
+				res->output = "";
+				list_add_tail(list, &res->list);
+			}
+		}
+	}
+
+	if (list_empty(list)) {
+		talloc_free(list);
+		list = NULL;
+	}
+
+	return list;
+}
+
+static unsigned int score_compile_tests(struct manifest *m,
+					void *check_result)
+{
+	struct list_head *list = check_result;
+	struct compile_tests_result *i;
+	unsigned int score = compile_tests.total_score;
+
+	list_for_each(list, i, list)
+		score--;
+	return score;
+}
+
+static const char *describe_compile_tests(struct manifest *m,
+					  void *check_result)
+{
+	struct list_head *list = check_result;
+	struct compile_tests_result *i;
+	char *descrip = talloc_strdup(list, "Compilation tests failed:\n");
+
+	list_for_each(list, i, list)
+		descrip = talloc_asprintf_append(descrip, "%s %s\n%s",
+						 i->filename, i->description,
+						 i->output);
+	return descrip;
+}
+
+struct ccanlint compile_tests = {
+	.key = "compile-tests",
+	.name = "Module tests compile",
+	.score = score_compile_tests,
+	.total_score = 1,
+	.check = do_compile_tests,
+	.describe = describe_compile_tests,
+	.can_run = can_build,
+};
+
+REGISTER_TEST(compile_tests, &compile_test_helpers, &build_objs, NULL);
diff --git a/tools/ccanlint/tests/examples_compile.c b/tools/ccanlint/tests/examples_compile.c
index 82b47962..e66379f5 100644
--- a/tools/ccanlint/tests/examples_compile.c
+++ b/tools/ccanlint/tests/examples_compile.c
@@ -506,6 +506,11 @@ static void *build_examples(struct manifest *m, bool keep,
 		/* This didn't work, so not a candidate for combining. */
 		prev = NULL;
 	}
+
+	if (strcmp(score->errors, "") == 0) {
+		talloc_free(score);
+		return NULL;
+	}
 	return score;
 }
 
diff --git a/tools/ccanlint/tests/has_tests.c b/tools/ccanlint/tests/has_tests.c
new file mode 100644
index 00000000..56eaf2a4
--- /dev/null
+++ b/tools/ccanlint/tests/has_tests.c
@@ -0,0 +1,138 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <ccan/talloc/talloc.h>
+
+static char test_is_not_dir[] = "test is not a directory";
+
+static void *check_has_tests(struct manifest *m,
+			     bool keep,
+			     unsigned int *timeleft)
+{
+	struct stat st;
+	char *test_dir = talloc_asprintf(m, "%s/test", m->dir);
+
+	if (lstat(test_dir, &st) != 0) {
+		if (errno != ENOENT)
+			err(1, "statting %s", test_dir);
+		return "You have no test directory";
+	}
+
+	if (!S_ISDIR(st.st_mode))
+		return test_is_not_dir;
+
+	if (list_empty(&m->api_tests)
+	    && list_empty(&m->run_tests)
+	    && list_empty(&m->compile_ok_tests)) {
+		if (list_empty(&m->compile_fail_tests))
+			return "You have no tests in the test directory";
+		else
+			return "You have no positive tests in the test directory";
+	}
+	return NULL;
+}
+
+static const char *describe_has_tests(struct manifest *m, void *check_result)
+{
+	return talloc_asprintf(m, "%s\n\n"
+        "CCAN modules have a directory called test/ which contains tests.\n"
+	"There are four kinds of tests: api, run, compile_ok and compile_fail:\n"
+	"you can tell which type of test a C file is by its name, eg 'run.c'\n"
+	"and 'run-simple.c' are both run tests.\n\n"
+
+	"The simplest kind of test is a run test, which must compile with no\n"
+	"warnings, and then run: it is expected to use libtap to report its\n"
+	"results in a simple and portable format.  It should #include the C\n"
+	"files from the module directly (so it can probe the internals): the\n"
+	"module will not be linked in.  The test will be run in a temporary\n"
+	"directory, with the test directory symlinked under test/.\n\n"
+
+	"api tests are just like a run test, except it is a guarantee of API\n"
+	"stability: this test should pass on all future versions of the\n"
+	"module.  They *are* linked to the module, since they should only\n"
+	"test the API, not the internal state.\n\n"
+
+	"compile_ok tests are a subset of run tests: they must compile and\n"
+	"link, but aren't run.\n\n"
+
+	"compile_fail tests are tests which should fail to compile (or emit\n"
+	"warnings) or link when FAIL is defined, but should compile and link\n"
+	"when it's not defined: this helps ensure unrelated errors don't make\n"
+	"compilation fail.\n\n"
+
+	"Note that the tests are not linked against the files in the\n"
+	"above: you should directly #include those C files you want.  This\n"
+	"allows access to static functions and use special effects inside\n"
+	"test files\n", (char *)check_result);
+}
+
+static void handle_no_tests(struct manifest *m, void *check_result)
+{
+	FILE *run;
+	struct ccan_file *i;
+
+	if (check_result == test_is_not_dir)
+		return;
+
+	if (!ask("Should I create a template test/run.c file for you?"))
+		return;
+
+	if (mkdir("test", 0700) != 0) {
+		if (errno != EEXIST)
+			err(1, "Creating test/ directory");
+	}
+
+	run = fopen("test/run.c", "w");
+	if (!run)
+		err(1, "Trying to create a test/run.c");
+
+	fputs("/* Include the main header first, to test it works */\n", run);
+	fprintf(run, "#include \"%s/%s.h\"\n", m->basename, m->basename);
+	fputs("/* Include the C files directly. */\n", run);
+	list_for_each(&m->c_files, i, list)
+		fprintf(run, "#include \"%s/%s\"\n", m->basename, i->name);
+	fputs("#include \"tap/tap.h\"\n", run);
+	fputs("\n", run);
+
+	fputs("int main(void)\n", run);
+	fputs("{\n", run);
+	fputs("\t/* This is how many tests you plan to run */\n", run);
+	fputs("\tplan_tests(3);\n", run);
+	fputs("\n", run);
+	fputs("\t/* Simple thing we expect to succeed */\n", run);
+	fputs("\tok1(some_test())\n", run);
+	fputs("\t/* Same, with an explicit description of the test. */\n", run);
+	fputs("\tok(some_test(), \"%s with no args should return 1\", \"some_test\")\n", run);
+	fputs("\t/* How to print out messages for debugging. */\n", run);
+	fputs("\tdiag(\"Address of some_test is %p\", &some_test)\n", run);
+	fputs("\t/* Conditional tests must be explicitly skipped. */\n", run);
+	fputs("#if HAVE_SOME_FEATURE\n", run);
+	fputs("\tok1(test_some_feature())\n", run);
+	fputs("#else\n", run);
+	fputs("\tskip(1, \"Don\'t have SOME_FEATURE\")\n", run);
+	fputs("#endif\n", run);
+	fputs("\n", run);
+	fputs("\t/* This exits depending on whether all tests passed */\n", run);
+	fputs("\treturn exit_status();\n", run);
+	fputs("}\n", run);
+
+	fclose(run);
+}
+
+struct ccanlint has_tests = {
+	.key = "has-tests",
+	.name = "Module has tests",
+	.check = check_has_tests,
+	.describe = describe_has_tests,
+	.total_score = 1,
+	.handle = handle_no_tests,
+};
+
+REGISTER_TEST(has_tests, NULL);
diff --git a/tools/ccanlint/tests/run-coverage.c b/tools/ccanlint/tests/run-coverage.c
index 886aebd5..a1bd8dea 100644
--- a/tools/ccanlint/tests/run-coverage.c
+++ b/tools/ccanlint/tests/run-coverage.c
@@ -209,4 +209,4 @@ struct ccanlint run_coverage_tests = {
 	.describe = describe_run_coverage_tests,
 };
 
-REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, NULL);
+REGISTER_TEST(run_coverage_tests, &compile_coverage_tests, &run_tests, NULL);
diff --git a/tools/ccanlint/tests/run_tests.c b/tools/ccanlint/tests/run_tests.c
new file mode 100644
index 00000000..559a6378
--- /dev/null
+++ b/tools/ccanlint/tests/run_tests.c
@@ -0,0 +1,126 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <limits.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <err.h>
+#include <string.h>
+#include <ctype.h>
+
+static const char *can_run(struct manifest *m)
+{
+	if (safe_mode)
+		return "Safe mode enabled";
+	return NULL;
+}
+
+struct run_tests_result {
+	struct list_node list;
+	struct ccan_file *file;
+	const char *output;
+};
+
+static void *do_run_tests(struct manifest *m,
+			  bool keep,
+			  unsigned int *timeleft)
+{
+	struct list_head *list = talloc(m, struct list_head);
+	struct run_tests_result *res;
+	struct ccan_file *i;
+	char *cmdout;
+
+	list_head_init(list);
+	run_tests.total_score = 0;
+
+	list_for_each(&m->run_tests, i, list) {
+		run_tests.total_score++;
+		cmdout = run_command(m, timeleft, i->compiled);
+		if (cmdout) {
+			res = talloc(list, struct run_tests_result);
+			res->file = i;
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		}
+	}
+
+	list_for_each(&m->api_tests, i, list) {
+		run_tests.total_score++;
+		cmdout = run_command(m, timeleft, i->compiled);
+		if (cmdout) {
+			res = talloc(list, struct run_tests_result);
+			res->file = i;
+			res->output = talloc_steal(res, cmdout);
+			list_add_tail(list, &res->list);
+		}
+	}
+
+	if (list_empty(list)) {
+		talloc_free(list);
+		list = NULL;
+	}
+
+	return list;
+}
+
+static unsigned int score_run_tests(struct manifest *m, void *check_result)
+{
+	struct list_head *list = check_result;
+	struct run_tests_result *i;
+	unsigned int score = run_tests.total_score;
+
+	list_for_each(list, i, list)
+		score--;
+	return score;
+}
+
+static const char *describe_run_tests(struct manifest *m,
+					  void *check_result)
+{
+	struct list_head *list = check_result;
+	char *descrip = talloc_strdup(check_result, "Running tests failed:\n");
+	struct run_tests_result *i;
+
+	list_for_each(list, i, list)
+		descrip = talloc_asprintf_append(descrip, "Running %s:\n%s",
+						 i->file->name, i->output);
+	return descrip;
+}
+
+/* Gcc's warn_unused_result is fascist bullshit. */
+#define doesnt_matter()
+
+static void run_under_debugger(struct manifest *m, void *check_result)
+{
+	char *command;
+	struct list_head *list = check_result;
+	struct run_tests_result *first;
+
+	if (!ask("Should I run the first failing test under the debugger?"))
+		return;
+
+	first = list_top(list, struct run_tests_result, list);
+	command = talloc_asprintf(m, "gdb -ex 'break tap.c:136' -ex 'run' %s",
+				  first->file->compiled);
+	if (system(command))
+		doesnt_matter();
+}
+
+struct ccanlint run_tests = {
+	.key = "run",
+	.name = "Module's run and api tests pass",
+	.score = score_run_tests,
+	.total_score = 1,
+	.check = do_run_tests,
+	.describe = describe_run_tests,
+	.can_run = can_run,
+	.handle = run_under_debugger
+};
+
+REGISTER_TEST(run_tests, &compile_tests, NULL);
diff --git a/tools/ccanlint/tests/run_tests_valgrind.c b/tools/ccanlint/tests/run_tests_valgrind.c
index 4b425ea2..caf8323c 100644
--- a/tools/ccanlint/tests/run_tests_valgrind.c
+++ b/tools/ccanlint/tests/run_tests_valgrind.c
@@ -41,6 +41,7 @@ static void *do_run_tests_vg(struct manifest *m,
 	char *cmdout;
 
 	list_head_init(list);
+	run_tests_vg.total_score = 0;
 
 	list_for_each(&m->run_tests, i, list) {
 		run_tests_vg.total_score++;
@@ -123,6 +124,7 @@ struct ccanlint run_tests_vg = {
 	.key = "valgrind-tests",
 	.name = "Module's run and api tests succeed under valgrind",
 	.score = score_run_tests_vg,
+	.total_score = 1,
 	.check = do_run_tests_vg,
 	.describe = describe_run_tests_vg,
 	.can_run = can_run_vg,