From 5378c864f9c37f39d906f599285da25a7db0c9fe Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 4 Nov 2010 13:55:15 +1030 Subject: [PATCH] ccanlint: try running example code. Comments of form "// [given x] outputs y" we can check the examples do as expected. --- tools/ccanlint/Makefile | 1 + tools/ccanlint/ccanlint.h | 1 + tools/ccanlint/file_analysis.c | 1 + tools/ccanlint/tests/examples_compile.c | 49 ++-- tools/ccanlint/tests/examples_run.c | 304 ++++++++++++++++++++++++ 5 files changed, 333 insertions(+), 23 deletions(-) create mode 100644 tools/ccanlint/tests/examples_run.c diff --git a/tools/ccanlint/Makefile b/tools/ccanlint/Makefile index fb0526db..974e7388 100644 --- a/tools/ccanlint/Makefile +++ b/tools/ccanlint/Makefile @@ -11,6 +11,7 @@ CORE_OBJS := tools/ccanlint/ccanlint.o \ ccan/str_talloc/str_talloc.o ccan/grab_file/grab_file.o \ ccan/btree/btree.o \ ccan/talloc/talloc.o ccan/noerr/noerr.o \ + ccan/foreach/foreach.o \ ccan/read_write_all/read_write_all.o \ ccan/opt/opt.o ccan/opt/usage.o ccan/opt/helpers.o ccan/opt/parse.o diff --git a/tools/ccanlint/ccanlint.h b/tools/ccanlint/ccanlint.h index 0bfb6a62..679b00ac 100644 --- a/tools/ccanlint/ccanlint.h +++ b/tools/ccanlint/ccanlint.h @@ -34,6 +34,7 @@ struct manifest { struct list_head other_files; struct list_head examples; + struct list_head mangled_examples; /* From tests/check_depends_exist.c */ struct list_head dep_dirs; diff --git a/tools/ccanlint/file_analysis.c b/tools/ccanlint/file_analysis.c index 9abd6228..b83e877e 100644 --- a/tools/ccanlint/file_analysis.c +++ b/tools/ccanlint/file_analysis.c @@ -180,6 +180,7 @@ struct manifest *get_manifest(const void *ctx, const char *dir) list_head_init(&m->other_test_files); list_head_init(&m->other_files); list_head_init(&m->examples); + list_head_init(&m->mangled_examples); list_head_init(&m->dep_dirs); olddir = talloc_getcwd(NULL); diff --git a/tools/ccanlint/tests/examples_compile.c b/tools/ccanlint/tests/examples_compile.c index e66379f5..f9a919a1 100644 --- a/tools/ccanlint/tests/examples_compile.c +++ b/tools/ccanlint/tests/examples_compile.c @@ -122,6 +122,7 @@ static char *compile(const void *ctx, "", lib_list(m), file->compiled); if (errmsg) { talloc_free(file->compiled); + file->compiled = NULL; return errmsg; } return NULL; @@ -429,6 +430,7 @@ static struct ccan_file *mangle_example(struct manifest *m, } close(fd); f->contents = talloc_steal(f, contents); + list_add(&m->mangled_examples, &f->list); return f; } @@ -450,35 +452,35 @@ static void *build_examples(struct manifest *m, bool keep, examples_compile.total_score++; /* Simplify our dumb parsing. */ strip_leading_whitespace(get_ccan_file_lines(i)); - ret = compile(score, m, i, keep); + ret = compile(i, m, i, keep); if (!ret) { prev = get_ccan_file_lines(i); score->score++; continue; } + /* Try standalone. */ + mangle1 = mangle_example(m, i, get_ccan_file_lines(i), keep); + ret1 = compile(i, m, mangle1, keep); + if (!ret1) { + prev = get_ccan_file_lines(i); + score->score++; + continue; + } + /* Try combining with previous (successful) example... */ if (prev) { char **new = combine(i, get_ccan_file_lines(i), prev); - mangle1 = mangle_example(m, i, new, keep); - ret1 = compile(score, m, mangle1, keep); - if (!ret1) { + mangle2 = mangle_example(m, i, new, keep); + ret2 = compile(i, m, mangle1, keep); + if (!ret2) { prev = new; score->score++; continue; } } - /* Try standalone. */ - mangle2 = mangle_example(m, i, get_ccan_file_lines(i), keep); - ret2 = compile(score, m, mangle2, keep); - if (!ret2) { - prev = get_ccan_file_lines(i); - score->score++; - continue; - } - score->errors = talloc_asprintf_append(score->errors, "%s: tried standalone example:\n" "%s\n" @@ -486,23 +488,24 @@ static void *build_examples(struct manifest *m, bool keep, i->name, get_ccan_file_contents(i), ret); - if (mangle1) { - score->errors = talloc_asprintf_append(score->errors, - "%s: tried combining with" - " previous example:\n" - "%s\n" - "Errors: %s\n\n", - i->name, - get_ccan_file_contents(mangle1), - ret1); - } score->errors = talloc_asprintf_append(score->errors, "%s: tried adding headers, wrappers:\n" "%s\n" "Errors: %s\n\n", i->name, + get_ccan_file_contents(mangle1), + ret1); + + if (mangle2) { + score->errors = talloc_asprintf_append(score->errors, + "%s\n" + "%s: tried combining with" + " previous example:\n" + "Errors: %s\n\n", + i->name, get_ccan_file_contents(mangle2), ret2); + } /* This didn't work, so not a candidate for combining. */ prev = NULL; } diff --git a/tools/ccanlint/tests/examples_run.c b/tools/ccanlint/tests/examples_run.c new file mode 100644 index 00000000..522feb0a --- /dev/null +++ b/tools/ccanlint/tests/examples_run.c @@ -0,0 +1,304 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static const char *can_run(struct manifest *m) +{ + if (safe_mode) + return "Safe mode enabled"; + return NULL; +} + +struct score { + unsigned int score; + char *errors; +}; + +/* Very dumb scanner, allocates %s-strings. */ +static bool scan_forv(const void *ctx, + const char *input, const char *fmt, const va_list *args) +{ + va_list ap; + bool ret; + + if (input[0] == '\0' || fmt[0] == '\0') + return input[0] == fmt[0]; + + va_copy(ap, *args); + + if (isspace(fmt[0])) { + /* One format space can swallow many input spaces */ + ret = false; + while (isspace(input[0])) { + if (scan_forv(ctx, ++input, fmt+1, &ap)) { + ret = true; + break; + } + } + } else if (fmt[0] != '%') { + if (toupper(input[0]) != toupper(fmt[0])) + ret = false; + else + ret = scan_forv(ctx, input+1, fmt+1, &ap); + } else { + char **p = va_arg(ap, char **); + unsigned int len; + + ret = false; + assert(fmt[1] == 's'); + for (len = 1; input[len-1]; len++) { + ret = scan_forv(ctx, input + len, fmt+2, &ap); + if (ret) { + *p = talloc_strndup(ctx, input, len); + ret = true; + break; + } + } + } + va_end(ap); + return ret; +} + +static bool scan_for(const void *ctx, const char *input, const char *fmt, ...) +{ + bool ret; + va_list ap; + + va_start(ap, fmt); + ret = scan_forv(ctx, input, fmt, &ap); + va_end(ap); + return ret; +} + +static char *find_expect(struct ccan_file *file, + char **lines, char **input, bool *exact, + unsigned *line) +{ + char *expect; + const char *fmt; + + for (; lines[*line]; (*line)++) { + char *p = lines[*line] + strspn(lines[*line], " \t"); + if (!strstarts(p, "//")) + continue; + p += strspn(p, "/ "); + foreach_ptr(fmt, + "given '%s', outputs '%s'", + "given '%s' outputs '%s'", + "given \"%s\", outputs \"%s\"", + "given \"%s\" outputs \"%s\"") { + if (scan_for(file, p, fmt, input, &expect)) { + *exact = true; + return expect; + } + } + + foreach_ptr(fmt, + "given '%s', output contains '%s'", + "given '%s' output contains '%s'", + "given \"%s\", output contains \"%s\"", + "given \"%s\" output contains \"%s\"") { + if (scan_for(file, p, fmt, input, &expect)) { + *exact = false; + return expect; + } + } + + foreach_ptr(fmt, "outputs '%s'", "outputs \"%s\"") { + if (scan_for(file, p, fmt, &expect)) { + *input = ""; + *exact = true; + return expect; + } + } + + foreach_ptr(fmt, + "given '%s', output contains '%s'", + "given '%s' output contains '%s'", + "given \"%s\", output contains \"%s\"", + "given \"%s\" output contains \"%s\"") { + if (scan_for(file, p, fmt, input, &expect)) { + *exact = false; + return expect; + } + } + + /* Unquoted versions... we can get this wrong! */ + foreach_ptr(fmt, + "given %s, outputs '%s'", + "given '%s', outputs %s", + "given %s, outputs \"%s\"", + "given \"%s\", outputs %s", + "given %s, outputs %s", + "given %s outputs '%s'", + "given '%s' outputs %s", + "given %s outputs \"%s\"", + "given \"%s\" outputs %s", + "given %s outputs %s") { + if (scan_for(file, p, fmt, input, &expect)) { + *exact = true; + return expect; + } + } + + foreach_ptr(fmt, + "given %s, output contains '%s'", + "given '%s', output contains %s", + "given %s, output contains \"%s\"", + "given \"%s\", output contains %s", + "given %s, output contains %s", + "given %s output contains '%s'", + "given '%s' output contains %s", + "given %s output contains \"%s\"", + "given \"%s\" output contains %s", + "given %s output contains %s") { + if (scan_for(file, p, fmt, input, &expect)) { + *exact = false; + return expect; + } + } + + foreach_ptr(fmt, + "outputs '%s'", + "outputs \"%s\"", + "outputs %s") { + if (scan_for(file, p, fmt, &expect)) { + *input = ""; + *exact = true; + return expect; + } + } + + foreach_ptr(fmt, + "output contains '%s'", + "output contains \"%s\"", + "output contains %s") { + if (scan_for(file, p, fmt, &expect)) { + *input = ""; + *exact = false; + return expect; + } + } + } + return NULL; +} + +static char *trim(char *string) +{ + while (strends(string, "\n")) + string[strlen(string)-1] = '\0'; + return string; +} + +static char *unexpected(struct ccan_file *i, const char *input, + const char *expect, bool exact) +{ + char *output, *cmd; + bool ok; + unsigned int default_time = default_timeout_ms; + + cmd = talloc_asprintf(i, "echo '%s' | %s %s", + input, i->compiled, input); + + output = run_with_timeout(i, cmd, &ok, &default_time); + if (!ok) + return talloc_asprintf(i, "Exited with non-zero status\n"); + + if (exact) { + if (streq(output, expect) || streq(trim(output), expect)) + return NULL; + } else { + if (strstr(output, expect)) + return NULL; + } + return output; +} + +static void *run_examples(struct manifest *m, bool keep, + unsigned int *timeleft) +{ + struct ccan_file *i; + struct list_head *list; + struct score *score = talloc(m, struct score); + + score->score = 0; + score->errors = talloc_strdup(score, ""); + + examples_run.total_score = 0; + foreach_ptr(list, &m->examples, &m->mangled_examples) { + list_for_each(list, i, list) { + char **lines, *expect, *input, *output; + unsigned int linenum = 0; + bool exact; + + if (i->compiled == NULL) + continue; + + lines = get_ccan_file_lines(i); + + for (expect = find_expect(i, lines, &input, &exact, + &linenum); + expect; + linenum++, + expect = find_expect(i, lines, &input, + &exact, &linenum)) { + examples_run.total_score++; + output = unexpected(i, input, expect, exact); + if (!output) + score->score++; + else { + score->errors = talloc_asprintf_append( + score->errors, + "%s: output '%s' didn't" + " %s '%s'\n", + i->name, output, + exact ? "match" : "contain", + expect); + } + } + } + } + + if (strcmp(score->errors, "") == 0) { + talloc_free(score); + return NULL; + } + return score; +} + +static unsigned int score_examples(struct manifest *m, void *check_result) +{ + struct score *score = check_result; + return score->score; +} + +static const char *describe(struct manifest *m, void *check_result) +{ + struct score *score = check_result; + if (verbose) + return talloc_asprintf(m, "Wrong output running examples:\n" + "%s", score->errors); + return NULL; +} + +struct ccanlint examples_run = { + .key = "examples-run", + .name = "Module examples with expected output give that output", + .score = score_examples, + .total_score = 3, /* This gets changed to # testable, if we run. */ + .check = run_examples, + .describe = describe, + .can_run = can_run, +}; + +REGISTER_TEST(examples_run, &examples_compile, NULL); -- 2.39.2