ccanlint: compile tests in parallel
authorRusty Russell <rusty@rustcorp.com.au>
Mon, 12 Sep 2011 04:50:39 +0000 (14:20 +0930)
committerRusty Russell <rusty@rustcorp.com.au>
Mon, 12 Sep 2011 04:50:39 +0000 (14:20 +0930)
Measuring ccanlint -d ccan/tdb2 on my dual core hyperthread laptop gives:
Before: 6:47
After: 5:59

tools/ccanlint/async.c
tools/ccanlint/ccanlint.h
tools/ccanlint/tests/tests_compile.c

index 32a67d29add913cbc2c4ae3f557b91534d87456b..9c7a442ee64fabcbd8a3a581a3f79e85ebbac725 100644 (file)
@@ -198,3 +198,18 @@ void *collect_command(bool *ok, char **output)
        talloc_free(c);
        return (void *)ctx;
 }
+
+/* Compile and link single C file, with object files, async. */
+void compile_and_link_async(const void *ctx, unsigned int time_ms,
+                           const char *cfile, const char *ccandir,
+                           const char *objs, const char *compiler,
+                           const char *cflags,
+                           const char *libs, const char *outfile)
+{
+       if (compile_verbose)
+               printf("Compiling and linking (async) %s\n", outfile);
+       run_command_async(ctx, time_ms,
+                         "%s %s -I%s -o %s %s %s %s",
+                         compiler, cflags,
+                         ccandir, outfile, cfile, objs, libs);
+}
index 13b10dcdd2a3539ba19ad2f700225816111ec6ff..452b44f7d8dfd6d93b14d322684873a304a0cacd 100644 (file)
@@ -239,6 +239,13 @@ void score_file_error(struct score *, struct ccan_file *f, unsigned line,
 void run_command_async(const void *ctx, unsigned int time_ms,
                       const char *fmt, ...);
 
+/* Async version of compile_and_link. */
+void compile_and_link_async(const void *ctx, unsigned int time_ms,
+                           const char *cfile, const char *ccandir,
+                           const char *objs, const char *compiler,
+                           const char *cflags,
+                           const char *libs, const char *outfile);
+
 /* Get results of a command, returning ctx (and free it). */
 void *collect_command(bool *ok, char **output);
 
index db5ea33e4f7467e9c9266674541625d71b8d1903..bfa412a68a14d55051588b15de3ed74213096f52 100644 (file)
@@ -95,30 +95,57 @@ static bool compile(const void *ctx,
        return true;
 }
 
+static void compile_async(const void *ctx,
+                         struct manifest *m,
+                         struct ccan_file *file,
+                         bool link_with_module,
+                         bool keep,
+                         enum compile_type ctype,
+                         unsigned int time_ms)
+{
+       char *flags;
+
+       file->compiled[ctype] = maybe_temp_file(ctx, "", keep, file->fullname);
+       flags = talloc_asprintf(ctx, "%s%s",
+                               cflags,
+                               ctype == COMPILE_NOFEAT
+                               ? " "REDUCE_FEATURES_FLAGS : "");
+
+       compile_and_link_async(file, time_ms, file->fullname, ccan_dir,
+                              test_obj_list(m, link_with_module, ctype, ctype),
+                              compiler, flags, lib_list(m, ctype),
+                              file->compiled[ctype]);
+}
+
 static void compile_tests(struct manifest *m, bool keep,
                          struct score *score,
-                         enum compile_type ctype)
+                         enum compile_type ctype,
+                         unsigned int time_ms)
 {
        char *cmdout;
        struct ccan_file *i;
        struct list_head *list;
-       bool errors = false, warnings = false;
+       bool errors = false, warnings = false, ok;
 
        foreach_ptr(list, &m->compile_ok_tests, &m->run_tests, &m->api_tests) {
                list_for_each(list, i, list) {
-                       if (!compile(score, m, i, false,
-                                    list == &m->api_tests, keep,
-                                    ctype, &cmdout)) {
-                               score_file_error(score, i, 0,
-                                                "Compile failed:\n%s",
-                                                cmdout);
-                               errors = true;
-                       } else if (!streq(cmdout, "")) {
-                               score_file_error(score, i, 0,
-                                                "Compile gave warnings:\n%s",
-                                                cmdout);
-                               warnings = true;
-                       }
+                       compile_async(score, m, i,
+                                     list == &m->api_tests, keep,
+                                     ctype, time_ms);
+               }
+       }
+
+       while ((i = collect_command(&ok, &cmdout)) != NULL) {
+               if (!ok) {
+                       score_file_error(score, i, 0,
+                                        "Compile failed:\n%s",
+                                        cmdout);
+                       errors = true;
+               } else if (!streq(cmdout, "")) {
+                       score_file_error(score, i, 0,
+                                        "Compile gave warnings:\n%s",
+                                        cmdout);
+                       warnings = true;
                }
        }
 
@@ -156,11 +183,12 @@ static void compile_tests(struct manifest *m, bool keep,
        score->score = score->total - warnings;
 }
 
+/* FIXME: If we time out, set *timeleft to 0 */
 static void do_compile_tests(struct manifest *m,
                             bool keep,
                             unsigned int *timeleft, struct score *score)
 {
-       compile_tests(m, keep, score, COMPILE_NORMAL);
+       compile_tests(m, keep, score, COMPILE_NORMAL, *timeleft);
 }
 
 struct ccanlint tests_compile = {
@@ -185,7 +213,7 @@ static void do_compile_tests_without_features(struct manifest *m,
                                              unsigned int *timeleft,
                                              struct score *score)
 {
-       compile_tests(m, keep, score, COMPILE_NOFEAT);
+       compile_tests(m, keep, score, COMPILE_NOFEAT, *timeleft);
 }
 
 struct ccanlint tests_compile_without_features = {