ccanlint: try running example code.

Comments of form "// [given x] outputs y" we can check the examples do as expected.

diff --git a/tools/ccanlint/Makefile b/tools/ccanlint/Makefile
index fb0526db52821375560f3eec65f486192a1add79..974e7388e714f17e0cab171c6480d8c8dbd2b6ed 100644 (file)
--- a/tools/ccanlint/Makefile
+++ b/tools/ccanlint/Makefile
@@ -11,6 +11,7 @@ CORE_OBJS := tools/ccanlint/ccanlint.o \
        ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o \
        ccan/btree/btree.o \
        ccan/talloc/talloc.o ccan/noerr/noerr.o \
+       ccan/foreach/foreach.o \
        ccan/read_write_all/read_write_all.o \
        ccan/opt/opt.o ccan/opt/usage.o ccan/opt/helpers.o ccan/opt/parse.o
 

diff --git a/tools/ccanlint/ccanlint.h b/tools/ccanlint/ccanlint.h
index 0bfb6a62cff5d88d99489b13b432c026beb8513e..679b00ac2bf2f87a02e4ac879fdbe1c9803b5611 100644 (file)
--- a/tools/ccanlint/ccanlint.h
+++ b/tools/ccanlint/ccanlint.h
@@ -34,6 +34,7 @@ struct manifest {
 
        struct list_head other_files;
        struct list_head examples;
+       struct list_head mangled_examples;
 
        /* From tests/check_depends_exist.c */
        struct list_head dep_dirs;

diff --git a/tools/ccanlint/file_analysis.c b/tools/ccanlint/file_analysis.c
index 9abd62288e6923307f68716d34ea7d7b2a96da04..b83e877e0455f6c3006537c90ea023e0034b2aaf 100644 (file)
--- a/tools/ccanlint/file_analysis.c
+++ b/tools/ccanlint/file_analysis.c
@@ -180,6 +180,7 @@ struct manifest *get_manifest(const void *ctx, const char *dir)
        list_head_init(&m->other_test_files);
        list_head_init(&m->other_files);
        list_head_init(&m->examples);
+       list_head_init(&m->mangled_examples);
        list_head_init(&m->dep_dirs);
 
        olddir = talloc_getcwd(NULL);

diff --git a/tools/ccanlint/tests/examples_compile.c b/tools/ccanlint/tests/examples_compile.c
index e66379f5e7fee45e9036a5cda09f664dac568b09..f9a919a1bfcf7d83748d026e79ec8eb2dd278601 100644 (file)
--- a/tools/ccanlint/tests/examples_compile.c
+++ b/tools/ccanlint/tests/examples_compile.c
@@ -122,6 +122,7 @@ static char *compile(const void *ctx,
                                  "", lib_list(m), file->compiled);
        if (errmsg) {
                talloc_free(file->compiled);
+               file->compiled = NULL;
                return errmsg;
        }
        return NULL;
@@ -429,6 +430,7 @@ static struct ccan_file *mangle_example(struct manifest *m,
        }
        close(fd);
        f->contents = talloc_steal(f, contents);
+       list_add(&m->mangled_examples, &f->list);
        return f;
 }
 
@@ -450,35 +452,35 @@ static void *build_examples(struct manifest *m, bool keep,
                examples_compile.total_score++;
                /* Simplify our dumb parsing. */
                strip_leading_whitespace(get_ccan_file_lines(i));
-               ret = compile(score, m, i, keep);
+               ret = compile(i, m, i, keep);
                if (!ret) {
                        prev = get_ccan_file_lines(i);
                        score->score++;
                        continue;
                }
 
+               /* Try standalone. */
+               mangle1 = mangle_example(m, i, get_ccan_file_lines(i), keep);
+               ret1 = compile(i, m, mangle1, keep);
+               if (!ret1) {
+                       prev = get_ccan_file_lines(i);
+                       score->score++;
+                       continue;
+               }
+
                /* Try combining with previous (successful) example... */
                if (prev) {
                        char **new = combine(i, get_ccan_file_lines(i), prev);
 
-                       mangle1 = mangle_example(m, i, new, keep);
-                       ret1 = compile(score, m, mangle1, keep);
-                       if (!ret1) {
+                       mangle2 = mangle_example(m, i, new, keep);
+                       ret2 = compile(i, m, mangle1, keep);
+                       if (!ret2) {
                                prev = new;
                                score->score++;
                                continue;
                        }
                }
 
-               /* Try standalone. */
-               mangle2 = mangle_example(m, i, get_ccan_file_lines(i), keep);
-               ret2 = compile(score, m, mangle2, keep);
-               if (!ret2) {
-                       prev = get_ccan_file_lines(i);
-                       score->score++;
-                       continue;
-               }
-
                score->errors = talloc_asprintf_append(score->errors,
                                       "%s: tried standalone example:\n"
                                       "%s\n"
@@ -486,23 +488,24 @@ static void *build_examples(struct manifest *m, bool keep,
                                       i->name,
                                       get_ccan_file_contents(i),
                                       ret);
-               if (mangle1) {
-                       score->errors = talloc_asprintf_append(score->errors,
-                                              "%s: tried combining with"
-                                              " previous example:\n"
-                                              "%s\n"
-                                              "Errors: %s\n\n",
-                                              i->name,
-                                              get_ccan_file_contents(mangle1),
-                                              ret1);
-               }
                score->errors = talloc_asprintf_append(score->errors,
                                       "%s: tried adding headers, wrappers:\n"
                                       "%s\n"
                                       "Errors: %s\n\n",
                                       i->name,
+                                      get_ccan_file_contents(mangle1),
+                                      ret1);
+
+               if (mangle2) {
+                       score->errors = talloc_asprintf_append(score->errors,
+                                      "%s\n"
+                                      "%s: tried combining with"
+                                      " previous example:\n"
+                                      "Errors: %s\n\n",
+                                      i->name,
                                       get_ccan_file_contents(mangle2),
                                       ret2);
+               }
                /* This didn't work, so not a candidate for combining. */
                prev = NULL;
        }

diff --git a/tools/ccanlint/tests/examples_run.c b/tools/ccanlint/tests/examples_run.c
new file mode 100644 (file)
index 0000000..522feb0
--- /dev/null
+++ b/tools/ccanlint/tests/examples_run.c
@@ -0,0 +1,304 @@
+#include <tools/ccanlint/ccanlint.h>
+#include <tools/tools.h>
+#include <ccan/talloc/talloc.h>
+#include <ccan/foreach/foreach.h>
+#include <ccan/str/str.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <stdint.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <assert.h>
+
+static const char *can_run(struct manifest *m)
+{
+       if (safe_mode)
+               return "Safe mode enabled";
+       return NULL;
+}
+
+struct score {
+       unsigned int score;
+       char *errors;
+};
+
+/* Very dumb scanner, allocates %s-strings. */
+static bool scan_forv(const void *ctx,
+                     const char *input, const char *fmt, const va_list *args)
+{
+       va_list ap;
+       bool ret;
+
+       if (input[0] == '\0' || fmt[0] == '\0')
+               return input[0] == fmt[0];
+
+       va_copy(ap, *args);
+
+       if (isspace(fmt[0])) {
+               /* One format space can swallow many input spaces */
+               ret = false;
+               while (isspace(input[0])) {
+                       if (scan_forv(ctx, ++input, fmt+1, &ap)) {
+                               ret = true;
+                               break;
+                       }
+               }
+       } else if (fmt[0] != '%') {
+               if (toupper(input[0]) != toupper(fmt[0]))
+                       ret = false;
+               else
+                       ret = scan_forv(ctx, input+1, fmt+1, &ap);
+       } else {
+               char **p = va_arg(ap, char **);
+               unsigned int len;
+
+               ret = false;
+               assert(fmt[1] == 's');
+               for (len = 1; input[len-1]; len++) {
+                       ret = scan_forv(ctx, input + len, fmt+2, &ap);
+                       if (ret) {
+                               *p = talloc_strndup(ctx, input, len);
+                               ret = true;
+                               break;
+                       }
+               }
+       }
+       va_end(ap);
+       return ret;
+}
+
+static bool scan_for(const void *ctx, const char *input, const char *fmt, ...)
+{
+       bool ret;
+       va_list ap;
+
+       va_start(ap, fmt);
+       ret = scan_forv(ctx, input, fmt, &ap);
+       va_end(ap);
+       return ret;
+}
+
+static char *find_expect(struct ccan_file *file,
+                        char **lines, char **input, bool *exact,
+                        unsigned *line)
+{
+       char *expect;
+       const char *fmt;
+
+       for (; lines[*line]; (*line)++) {
+               char *p = lines[*line] + strspn(lines[*line], " \t");
+               if (!strstarts(p, "//"))
+                       continue;
+               p += strspn(p, "/ ");
+               foreach_ptr(fmt,
+                           "given '%s', outputs '%s'",
+                           "given '%s' outputs '%s'",
+                           "given \"%s\", outputs \"%s\"",
+                           "given \"%s\" outputs \"%s\"") {
+                       if (scan_for(file, p, fmt, input, &expect)) {
+                               *exact = true;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt,
+                           "given '%s', output contains '%s'",
+                           "given '%s' output contains '%s'",
+                           "given \"%s\", output contains \"%s\"",
+                           "given \"%s\" output contains \"%s\"") {
+                       if (scan_for(file, p, fmt, input, &expect)) {
+                               *exact = false;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt, "outputs '%s'", "outputs \"%s\"") {
+                       if (scan_for(file, p, fmt, &expect)) {
+                               *input = "";
+                               *exact = true;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt,
+                           "given '%s', output contains '%s'",
+                           "given '%s' output contains '%s'",
+                           "given \"%s\", output contains \"%s\"",
+                           "given \"%s\" output contains \"%s\"") {
+                       if (scan_for(file, p, fmt, input, &expect)) {
+                               *exact = false;
+                               return expect;
+                       }
+               }
+
+               /* Unquoted versions... we can get this wrong! */
+               foreach_ptr(fmt,
+                           "given %s, outputs '%s'",
+                           "given '%s', outputs %s",
+                           "given %s, outputs \"%s\"",
+                           "given \"%s\", outputs %s",
+                           "given %s, outputs %s",
+                           "given %s outputs '%s'",
+                           "given '%s' outputs %s",
+                           "given %s outputs \"%s\"",
+                           "given \"%s\" outputs %s",
+                           "given %s outputs %s") {
+                       if (scan_for(file, p, fmt, input, &expect)) {
+                               *exact = true;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt,
+                           "given %s, output contains '%s'",
+                           "given '%s', output contains %s",
+                           "given %s, output contains \"%s\"",
+                           "given \"%s\", output contains %s",
+                           "given %s, output contains %s",
+                           "given %s output contains '%s'",
+                           "given '%s' output contains %s",
+                           "given %s output contains \"%s\"",
+                           "given \"%s\" output contains %s",
+                           "given %s output contains %s") {
+                       if (scan_for(file, p, fmt, input, &expect)) {
+                               *exact = false;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt,
+                           "outputs '%s'",
+                           "outputs \"%s\"",
+                           "outputs %s") {
+                       if (scan_for(file, p, fmt, &expect)) {
+                               *input = "";
+                               *exact = true;
+                               return expect;
+                       }
+               }
+
+               foreach_ptr(fmt,
+                           "output contains '%s'",
+                           "output contains \"%s\"",
+                           "output contains %s") {
+                       if (scan_for(file, p, fmt, &expect)) {
+                               *input = "";
+                               *exact = false;
+                               return expect;
+                       }
+               }
+       }               
+       return NULL;
+}
+
+static char *trim(char *string)
+{
+       while (strends(string, "\n"))
+              string[strlen(string)-1] = '\0';
+       return string;
+}
+
+static char *unexpected(struct ccan_file *i, const char *input,
+                       const char *expect, bool exact)
+{
+       char *output, *cmd;
+       bool ok;
+       unsigned int default_time = default_timeout_ms;
+
+       cmd = talloc_asprintf(i, "echo '%s' | %s %s",
+                             input, i->compiled, input);
+
+       output = run_with_timeout(i, cmd, &ok, &default_time);
+       if (!ok)
+               return talloc_asprintf(i, "Exited with non-zero status\n");
+
+       if (exact) {
+               if (streq(output, expect) || streq(trim(output), expect))
+                       return NULL;
+       } else {
+               if (strstr(output, expect))
+                       return NULL;
+       }
+       return output;
+}
+
+static void *run_examples(struct manifest *m, bool keep,
+                         unsigned int *timeleft)
+{
+       struct ccan_file *i;
+       struct list_head *list;
+       struct score *score = talloc(m, struct score);
+
+       score->score = 0;
+       score->errors = talloc_strdup(score, "");
+
+       examples_run.total_score = 0;
+       foreach_ptr(list, &m->examples, &m->mangled_examples) {
+               list_for_each(list, i, list) {
+                       char **lines, *expect, *input, *output;
+                       unsigned int linenum = 0;
+                       bool exact;
+
+                       if (i->compiled == NULL)
+                               continue;
+
+                       lines = get_ccan_file_lines(i);
+
+                       for (expect = find_expect(i, lines, &input, &exact,
+                                                 &linenum);
+                            expect;
+                            linenum++,
+                                    expect = find_expect(i, lines, &input,
+                                                         &exact, &linenum)) {
+                               examples_run.total_score++;
+                               output = unexpected(i, input, expect, exact);
+                               if (!output)
+                                       score->score++;
+                               else {
+                                       score->errors = talloc_asprintf_append(
+                                               score->errors,
+                                               "%s: output '%s' didn't"
+                                               " %s '%s'\n",
+                                               i->name, output,
+                                               exact ? "match" : "contain",
+                                               expect);
+                               }
+                       }
+               }
+       }
+
+       if (strcmp(score->errors, "") == 0) {
+               talloc_free(score);
+               return NULL;
+       }
+       return score;
+}
+
+static unsigned int score_examples(struct manifest *m, void *check_result)
+{
+       struct score *score = check_result;
+       return score->score;
+}
+
+static const char *describe(struct manifest *m, void *check_result)
+{
+       struct score *score = check_result;
+       if (verbose)
+               return talloc_asprintf(m, "Wrong output running examples:\n"
+                                      "%s", score->errors);
+       return NULL;
+}
+
+struct ccanlint examples_run = {
+       .key = "examples-run",
+       .name = "Module examples with expected output give that output",
+       .score = score_examples,
+       .total_score = 3, /* This gets changed to # testable, if we run. */
+       .check = run_examples,
+       .describe = describe,
+       .can_run = can_run,
+};
+
+REGISTER_TEST(examples_run, &examples_compile, NULL);