From: Rusty Russell <rusty@rustcorp.com.au>
Date: Fri, 9 Apr 2010 03:51:26 +0000 (+0930)
Subject: ccanlint: timeout, and implement -t option for quicker tests.
X-Git-Url: http://git.ozlabs.org/?p=ccan;a=commitdiff_plain;h=304652023042670b3173de0ad5dc0eb7c836618c

ccanlint: timeout, and implement -t option for quicker tests.
---

diff --git a/tools/ccanlint/ccanlint.c b/tools/ccanlint/ccanlint.c
index ee8c8b78..a65c6410 100644
--- a/tools/ccanlint/ccanlint.c
+++ b/tools/ccanlint/ccanlint.c
@@ -37,16 +37,18 @@ static LIST_HEAD(normal_tests);
 static LIST_HEAD(finished_tests);
 bool safe_mode = false;
 static struct btree *exclude;
+static bool fastmode = false;
 
 static void usage(const char *name)
 {
-	fprintf(stderr, "Usage: %s [-s] [-n] [-v] [-d <dirname>]\n"
+	fprintf(stderr, "Usage: %s [-s] [-n] [-v] [-t] [-d <dirname>]\n"
 		"   -v: verbose mode\n"
 		"   -s: simply give one line summary\n"
 		"   -d: use this directory instead of the current one\n"
 		"   -n: do not compile anything\n"
 		"   -l: list tests ccanlint performs\n"
-		"   -x: exclude tests (e.g. -x trailing_whitespace,valgrind)\n",
+		"   -x: exclude tests (e.g. -x trailing_whitespace,valgrind)\n"
+		"   -t: ignore (terminate) tests that are slow\n",
 		name);
 	exit(1);
 }
@@ -100,7 +102,7 @@ static bool run_test(struct ccanlint *i,
 		     struct manifest *m)
 {
 	void *result;
-	unsigned int this_score;
+	unsigned int this_score, timeleft;
 	const struct dependent *d;
 	const char *skip;
 
@@ -109,7 +111,9 @@ static bool run_test(struct ccanlint *i,
 		d->dependent->num_depends--;
 
 	skip = should_skip(m, i);
+
 	if (skip) {
+	skip:
 		if (verbose)
 			printf("  %s: skipped (%s)\n", i->name, skip);
 
@@ -126,7 +130,12 @@ static bool run_test(struct ccanlint *i,
 		return true;
 	}
 
-	result = i->check(m);
+	timeleft = fastmode ? 1000 : default_timeout_ms;
+	result = i->check(m, &timeleft);
+	if (fastmode && timeleft == 0) {
+		skip = "timeout";
+		goto skip;
+	}
 	if (!result) {
 		if (verbose) {
 			printf("  %s: OK", i->name);
@@ -311,7 +320,7 @@ int main(int argc, char *argv[])
 
 	/* I'd love to use long options, but that's not standard. */
 	/* FIXME: getopt_long ccan package? */
-	while ((c = getopt(argc, argv, "sd:vnlx:")) != -1) {
+	while ((c = getopt(argc, argv, "sd:vnlx:t")) != -1) {
 		switch (c) {
 		case 'd':
 			dir = optarg;
@@ -336,6 +345,9 @@ int main(int argc, char *argv[])
 			for (i = 0; exclude_strs[i]; i++)
 				btree_insert(exclude, exclude_strs[i]);
 		} break;
+		case 't':
+			fastmode = true;
+			break;
 		default:
 			usage(argv[0]);
 		}
diff --git a/tools/ccanlint/ccanlint.h b/tools/ccanlint/ccanlint.h
index cf277393..2ddbf37a 100644
--- a/tools/ccanlint/ccanlint.h
+++ b/tools/ccanlint/ccanlint.h
@@ -52,8 +52,9 @@ struct ccanlint {
 	/* Can we run this test?  Return string explaining why, if not. */
 	const char *(*can_run)(struct manifest *m);
 
-	/* If this returns non-NULL, it means the check failed. */
-	void *(*check)(struct manifest *m);
+	/* If this returns non-NULL, it means the check failed.
+	 * If timeleft is set to 0, means it timed out. */
+	void *(*check)(struct manifest *m, unsigned int *timeleft);
 
 	/* The non-NULL return from check is passed to one of these: */
 
diff --git a/tools/ccanlint/compulsory_tests/build.c b/tools/ccanlint/compulsory_tests/build.c
index efc0b27a..ec9e6de1 100644
--- a/tools/ccanlint/compulsory_tests/build.c
+++ b/tools/ccanlint/compulsory_tests/build.c
@@ -33,7 +33,7 @@ static char *obj_list(const struct manifest *m)
 	return list;
 }
 
-static void *do_build(struct manifest *m)
+static void *do_build(struct manifest *m, unsigned int *timeleft)
 {
 	char *filename, *err;
 
diff --git a/tools/ccanlint/compulsory_tests/build_objs.c b/tools/ccanlint/compulsory_tests/build_objs.c
index 0768bc6f..619c6008 100644
--- a/tools/ccanlint/compulsory_tests/build_objs.c
+++ b/tools/ccanlint/compulsory_tests/build_objs.c
@@ -21,7 +21,7 @@ static const char *can_build(struct manifest *m)
 	return NULL;
 }
 
-static void *check_objs_build(struct manifest *m)
+static void *check_objs_build(struct manifest *m, unsigned int *timeleft)
 {
 	char *report = NULL;
 	struct ccan_file *i;
diff --git a/tools/ccanlint/compulsory_tests/check_build.c b/tools/ccanlint/compulsory_tests/check_build.c
index 820cc9fe..2098f0d2 100644
--- a/tools/ccanlint/compulsory_tests/check_build.c
+++ b/tools/ccanlint/compulsory_tests/check_build.c
@@ -44,7 +44,7 @@ static char *lib_list(const struct manifest *m)
 	return ret;
 }
 
-static void *check_use_build(struct manifest *m)
+static void *check_use_build(struct manifest *m, unsigned int *timeleft)
 {
 	char *contents;
 	char *tmpfile, *err;
diff --git a/tools/ccanlint/compulsory_tests/check_depends_built.c b/tools/ccanlint/compulsory_tests/check_depends_built.c
index a3e42f26..c261315d 100644
--- a/tools/ccanlint/compulsory_tests/check_depends_built.c
+++ b/tools/ccanlint/compulsory_tests/check_depends_built.c
@@ -35,7 +35,7 @@ static bool expect_obj_file(const char *dir)
 	return has_c_files;
 }
 
-static void *check_depends_built(struct manifest *m)
+static void *check_depends_built(struct manifest *m, unsigned int *timeleft)
 {
 	struct ccan_file *i;
 	struct stat st;
diff --git a/tools/ccanlint/compulsory_tests/check_depends_exist.c b/tools/ccanlint/compulsory_tests/check_depends_exist.c
index aef2a24b..bb6067ed 100644
--- a/tools/ccanlint/compulsory_tests/check_depends_exist.c
+++ b/tools/ccanlint/compulsory_tests/check_depends_exist.c
@@ -33,7 +33,7 @@ static char *add_dep(char *sofar, struct manifest *m, const char *dep)
 	return sofar;
 }
 
-static void *check_depends_exist(struct manifest *m)
+static void *check_depends_exist(struct manifest *m, unsigned int *timeleft)
 {
 	unsigned int i;
 	char *report = NULL;
diff --git a/tools/ccanlint/compulsory_tests/check_includes_build.c b/tools/ccanlint/compulsory_tests/check_includes_build.c
index 2962765a..f50e5430 100644
--- a/tools/ccanlint/compulsory_tests/check_includes_build.c
+++ b/tools/ccanlint/compulsory_tests/check_includes_build.c
@@ -22,7 +22,7 @@ static const char *can_build(struct manifest *m)
 	return NULL;
 }
 
-static void *check_includes_build(struct manifest *m)
+static void *check_includes_build(struct manifest *m, unsigned int *timeleft)
 {
 	char *contents;
 	char *tmpfile, *err;
diff --git a/tools/ccanlint/compulsory_tests/compile_test_helpers.c b/tools/ccanlint/compulsory_tests/compile_test_helpers.c
index 127d972b..d229183a 100644
--- a/tools/ccanlint/compulsory_tests/compile_test_helpers.c
+++ b/tools/ccanlint/compulsory_tests/compile_test_helpers.c
@@ -32,7 +32,7 @@ static char *compile(struct manifest *m, struct ccan_file *cfile)
 	return err;
 }
 
-static void *do_compile_test_helpers(struct manifest *m)
+static void *do_compile_test_helpers(struct manifest *m, unsigned int *timeleft)
 {
 	char *cmdout = NULL;
 	struct ccan_file *i;
diff --git a/tools/ccanlint/compulsory_tests/compile_tests.c b/tools/ccanlint/compulsory_tests/compile_tests.c
index 89ba29b9..1e184f2d 100644
--- a/tools/ccanlint/compulsory_tests/compile_tests.c
+++ b/tools/ccanlint/compulsory_tests/compile_tests.c
@@ -80,7 +80,7 @@ struct compile_tests_result {
 	const char *output;
 };
 
-static void *do_compile_tests(struct manifest *m)
+static void *do_compile_tests(struct manifest *m, unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	char *cmdout;
diff --git a/tools/ccanlint/compulsory_tests/has_info.c b/tools/ccanlint/compulsory_tests/has_info.c
index 4b6687e4..97cad594 100644
--- a/tools/ccanlint/compulsory_tests/has_info.c
+++ b/tools/ccanlint/compulsory_tests/has_info.c
@@ -11,7 +11,7 @@
 #include <string.h>
 #include <ccan/noerr/noerr.h>
 
-static void *check_has_info(struct manifest *m)
+static void *check_has_info(struct manifest *m, unsigned int *timeleft)
 {
 	if (m->info_file)
 		return NULL;
diff --git a/tools/ccanlint/compulsory_tests/has_main_header.c b/tools/ccanlint/compulsory_tests/has_main_header.c
index c2c82382..dafb41a6 100644
--- a/tools/ccanlint/compulsory_tests/has_main_header.c
+++ b/tools/ccanlint/compulsory_tests/has_main_header.c
@@ -12,7 +12,7 @@
 #include <ccan/talloc/talloc.h>
 #include <ccan/noerr/noerr.h>
 
-static void *check_has_main_header(struct manifest *m)
+static void *check_has_main_header(struct manifest *m, unsigned int *timeleft)
 {
 	struct ccan_file *f;
 
diff --git a/tools/ccanlint/compulsory_tests/has_tests.c b/tools/ccanlint/compulsory_tests/has_tests.c
index 0fa11aaa..df34a255 100644
--- a/tools/ccanlint/compulsory_tests/has_tests.c
+++ b/tools/ccanlint/compulsory_tests/has_tests.c
@@ -12,7 +12,7 @@
 
 static char test_is_not_dir[] = "test is not a directory";
 
-static void *check_has_tests(struct manifest *m)
+static void *check_has_tests(struct manifest *m, unsigned int *timeleft)
 {
 	struct stat st;
 	char *test_dir = talloc_asprintf(m, "%s/test", m->dir);
diff --git a/tools/ccanlint/compulsory_tests/run_tests.c b/tools/ccanlint/compulsory_tests/run_tests.c
index 581654a1..52c5fb10 100644
--- a/tools/ccanlint/compulsory_tests/run_tests.c
+++ b/tools/ccanlint/compulsory_tests/run_tests.c
@@ -27,7 +27,7 @@ struct run_tests_result {
 	const char *output;
 };
 
-static void *do_run_tests(struct manifest *m)
+static void *do_run_tests(struct manifest *m, unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	struct run_tests_result *res;
@@ -47,8 +47,7 @@ static void *do_run_tests(struct manifest *m)
 
 	list_for_each(&m->run_tests, i, list) {
 		run_tests.total_score++;
-		/* FIXME: timeout here */
-		cmdout = run_command(m, i->compiled);
+		cmdout = run_command(m, timeleft, i->compiled);
 		if (cmdout) {
 			res = talloc(list, struct run_tests_result);
 			res->file = i;
@@ -59,8 +58,7 @@ static void *do_run_tests(struct manifest *m)
 
 	list_for_each(&m->api_tests, i, list) {
 		run_tests.total_score++;
-		/* FIXME: timeout here */
-		cmdout = run_command(m, i->compiled);
+		cmdout = run_command(m, timeleft, i->compiled);
 		if (cmdout) {
 			res = talloc(list, struct run_tests_result);
 			res->file = i;
diff --git a/tools/ccanlint/tests/has_info_documentation.c b/tools/ccanlint/tests/has_info_documentation.c
index 4d440e6d..3ca844c8 100644
--- a/tools/ccanlint/tests/has_info_documentation.c
+++ b/tools/ccanlint/tests/has_info_documentation.c
@@ -22,7 +22,8 @@ struct info_docs
 	bool example;
 };
 
-static void *check_has_info_documentation(struct manifest *m)
+static void *check_has_info_documentation(struct manifest *m,
+					  unsigned int *timeleft)
 {
 	struct list_head *infodocs = get_ccan_file_docs(m->info_file);
 	struct doc_section *d;
diff --git a/tools/ccanlint/tests/idempotent.c b/tools/ccanlint/tests/idempotent.c
index 34b455c0..05f4c593 100644
--- a/tools/ccanlint/tests/idempotent.c
+++ b/tools/ccanlint/tests/idempotent.c
@@ -112,7 +112,7 @@ static char *report_idem(struct ccan_file *f, char *sofar)
 	return sofar;
 }
 
-static void *check_idempotent(struct manifest *m)
+static void *check_idempotent(struct manifest *m, unsigned int *timeleft)
 {
 	struct ccan_file *f;
 	char *report = NULL;
diff --git a/tools/ccanlint/tests/run_tests_valgrind.c b/tools/ccanlint/tests/run_tests_valgrind.c
index 5819b5eb..3c6fb81c 100644
--- a/tools/ccanlint/tests/run_tests_valgrind.c
+++ b/tools/ccanlint/tests/run_tests_valgrind.c
@@ -17,7 +17,8 @@
 /* Note: we already test safe_mode in run_tests.c */
 static const char *can_run_vg(struct manifest *m)
 {
-	char *output = run_command(m, "valgrind -q true");
+	unsigned int timeleft = default_timeout_ms;
+	char *output = run_command(m, &timeleft, "valgrind -q true");
 
 	if (output)
 		return talloc_asprintf(m, "No valgrind support: %s", output);
@@ -30,7 +31,7 @@ struct run_tests_result {
 	const char *output;
 };
 
-static void *do_run_tests_vg(struct manifest *m)
+static void *do_run_tests_vg(struct manifest *m, unsigned int *timeleft)
 {
 	struct list_head *list = talloc(m, struct list_head);
 	struct run_tests_result *res;
@@ -50,8 +51,8 @@ static void *do_run_tests_vg(struct manifest *m)
 
 	list_for_each(&m->run_tests, i, list) {
 		run_tests_vg.total_score++;
-		/* FIXME: timeout here */
-		cmdout = run_command(m, "valgrind -q %s", i->compiled);
+		cmdout = run_command(m, timeleft,
+				     "valgrind -q %s", i->compiled);
 		if (cmdout) {
 			res = talloc(list, struct run_tests_result);
 			res->file = i;
@@ -62,8 +63,8 @@ static void *do_run_tests_vg(struct manifest *m)
 
 	list_for_each(&m->api_tests, i, list) {
 		run_tests_vg.total_score++;
-		/* FIXME: timeout here */
-		cmdout = run_command(m, "valgrind -q %s", i->compiled);
+		cmdout = run_command(m, timeleft,
+				     "valgrind -q %s", i->compiled);
 		if (cmdout) {
 			res = talloc(list, struct run_tests_result);
 			res->file = i;
diff --git a/tools/ccanlint/tests/trailing_whitespace.c b/tools/ccanlint/tests/trailing_whitespace.c
index 6b831a72..0fdd0cf8 100644
--- a/tools/ccanlint/tests/trailing_whitespace.c
+++ b/tools/ccanlint/tests/trailing_whitespace.c
@@ -19,7 +19,8 @@ static char *report_on_trailing_whitespace(const char *line)
 	return talloc_asprintf(line, "'%s'", line);
 }
 
-static void *check_trailing_whitespace(struct manifest *m)
+static void *check_trailing_whitespace(struct manifest *m,
+				       unsigned int *timeleft)
 {
 	char *report;
 
diff --git a/tools/compile.c b/tools/compile.c
index f166512e..089a9ab0 100644
--- a/tools/compile.c
+++ b/tools/compile.c
@@ -7,7 +7,7 @@ char *link_objects(const void *ctx, const char *objs, char **errmsg)
 {
 	char *file = temp_file(ctx, ".o");
 
-	*errmsg = run_command(ctx, "ld -r -o %s %s", file, objs);
+	*errmsg = run_command(ctx, NULL, "ld -r -o %s %s", file, objs);
 	if (*errmsg) {
 		talloc_free(file);
 		return NULL;
@@ -21,7 +21,7 @@ char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
 {
 	char *file = temp_file(ctx, ".o");
 
-	*errmsg = run_command(ctx, "cc " CFLAGS " -I%s -c -o %s %s",
+	*errmsg = run_command(ctx, NULL, "cc " CFLAGS " -I%s -c -o %s %s",
 			      ccandir, file, cfile);
 	if (*errmsg) {
 		talloc_free(file);
@@ -38,7 +38,7 @@ char *compile_and_link(const void *ctx, const char *cfile, const char *ccandir,
 {
 	char *file = temp_file(ctx, "");
 
-	*errmsg = run_command(ctx, "cc " CFLAGS " -I%s %s -o %s %s %s %s",
+	*errmsg = run_command(ctx, NULL, "cc " CFLAGS " -I%s %s -o %s %s %s %s",
 			      ccandir, extra_cflags, file, cfile, objs, libs);
 	if (*errmsg) {
 		talloc_free(file);
diff --git a/tools/tools.c b/tools/tools.c
index 18cc1b84..6e19f961 100644
--- a/tools/tools.c
+++ b/tools/tools.c
@@ -2,8 +2,11 @@
 #include <ccan/grab_file/grab_file.h>
 #include <ccan/noerr/noerr.h>
 #include <ccan/read_write_all/read_write_all.h>
+#include <ccan/noerr/noerr.h>
 #include <sys/stat.h>
 #include <sys/types.h>
+#include <sys/time.h>
+#include <sys/wait.h>
 #include <fcntl.h>
 #include <string.h>
 #include <unistd.h>
@@ -15,6 +18,9 @@
 static char *tmpdir = NULL;
 static unsigned int count;
 
+/* Ten minutes. */
+const unsigned int default_timeout_ms = 10 * 60 * 1000;
+
 char *talloc_basename(const void *ctx, const char *dir)
 {
 	char *p = strrchr(dir, '/');
@@ -51,31 +57,106 @@ char *talloc_getcwd(const void *ctx)
 	return cwd;
 }
 
+static char *run_with_timeout(const void *ctx,
+			      const char *cmd,
+			      bool *ok,
+			      unsigned *timeout_ms)
+{
+	pid_t pid;
+	int p[2];
+	char *ret;
+	int status, ms;
+	struct timeval start, end;
+
+	*ok = false;
+	if (pipe(p) != 0)
+		return talloc_asprintf(ctx, "Failed to create pipe: %s",
+				       strerror(errno));
+
+	pid = fork();
+	if (pid == -1) {
+		close_noerr(p[0]);
+		close_noerr(p[1]);
+		return talloc_asprintf(ctx, "Failed to fork: %s",
+				       strerror(errno));
+		return NULL;
+	}
+
+	if (pid == 0) {
+		struct itimerval itim;
+
+		if (dup2(p[1], STDOUT_FILENO) != STDOUT_FILENO
+		    || dup2(p[1], STDERR_FILENO) != STDERR_FILENO
+		    || close(p[0]) != 0
+		    || close(STDIN_FILENO) != 0
+		    || open("/dev/null", O_RDONLY) != STDIN_FILENO)
+			exit(128);
+
+		itim.it_interval.tv_sec = itim.it_interval.tv_usec = 0;
+		itim.it_value.tv_sec = *timeout_ms / 1000;
+		itim.it_value.tv_usec = (*timeout_ms % 1000) * 1000;
+		setitimer(ITIMER_REAL, &itim, NULL);
+
+		status = system(cmd);
+		if (WIFEXITED(status))
+			exit(WEXITSTATUS(status));
+		/* Here's a hint... */
+		exit(128 + WTERMSIG(status));
+	}
+
+	close(p[1]);
+	gettimeofday(&start, NULL);
+	ret = grab_fd(ctx, p[0], NULL);
+	/* This shouldn't fail... */
+	if (waitpid(pid, &status, 0) != pid)
+		err(1, "Failed to wait for child");
+
+	gettimeofday(&end, NULL);
+	if (WIFSIGNALED(status)) {
+		*timeout_ms = 0;
+		return ret;
+	}
+	if (end.tv_usec < start.tv_usec) {
+		end.tv_usec += 1000000;
+		end.tv_sec--;
+	}
+	ms = (end.tv_sec - start.tv_sec) * 1000
+		+ (end.tv_usec - start.tv_usec) / 1000;
+	if (ms > *timeout_ms)
+		*timeout_ms = 0;
+	else
+		*timeout_ms -= ms;
+
+	*ok = (WEXITSTATUS(status) == 0);
+	return ret;
+}
+
 /* Returns output if command fails. */
-char *run_command(const void *ctx, const char *fmt, ...)
+char *run_command(const void *ctx, unsigned int *time_ms, const char *fmt, ...)
 {
 	va_list ap;
 	char *cmd, *contents;
-	FILE *pipe;
+	bool ok;
+	unsigned int default_time = default_timeout_ms;
+
+	if (!time_ms)
+		time_ms = &default_time;
 
 	va_start(ap, fmt);
 	cmd = talloc_vasprintf(ctx, fmt, ap);
 	va_end(ap);
 
-	/* Ensure stderr gets to us too. */
-	cmd = talloc_asprintf_append(cmd, " 2>&1");
-
-	pipe = popen(cmd, "r");
-	if (!pipe)
-		return talloc_asprintf(ctx, "Failed to run '%s'", cmd);
-
-	contents = grab_fd(cmd, fileno(pipe), NULL);
-	if (pclose(pipe) != 0)
-		return talloc_asprintf(ctx, "Running '%s':\n%s",
-				       cmd, contents);
+	contents = run_with_timeout(ctx, cmd, &ok, time_ms);
+	if (ok) {
+		talloc_free(contents);
+		return NULL;
+	}
 
-	talloc_free(cmd);
-	return NULL;
+	if (!contents)
+		err(1, "Problem running child");
+	if (*time_ms == 0)
+		talloc_asprintf_append(contents, "\n== TIMED OUT ==\n");
+	return contents;
 }
 
 static int unlink_all(char *dir)
diff --git a/tools/tools.h b/tools/tools.h
index 34c2f99f..9a4082db 100644
--- a/tools/tools.h
+++ b/tools/tools.h
@@ -27,7 +27,7 @@ char **get_libs(const void *ctx, const char *dir,
 char *talloc_basename(const void *ctx, const char *dir);
 char *talloc_dirname(const void *ctx, const char *dir);
 char *talloc_getcwd(const void *ctx);
-char *run_command(const void *ctx, const char *fmt, ...);
+char *run_command(const void *ctx, unsigned int *time_ms, const char *fmt, ...);
 char *temp_file(const void *ctx, const char *extension);
 bool move_file(const char *oldname, const char *newname);
 
@@ -45,4 +45,8 @@ char *compile_object(const void *ctx, const char *cfile, const char *ccandir,
 char *compile_and_link(const void *ctx, const char *cfile, const char *ccandir,
 		       const char *objs, const char *extra_cflags,
 		       const char *libs, char **errmsg);
+
+/* Default wait for run_command.  Should never time out. */
+extern const unsigned int default_timeout_ms;
+
 #endif /* CCAN_TOOLS_H */