From b87f63cb8228c8fcbf760defdcf9e4c332da7667 Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Mon, 28 Sep 2015 12:44:40 +0930 Subject: [PATCH] ccanlint: enhance and streamline "output" testing lines. 1) Require "" around input 2) Make them optional around output: if not there, loose match whitespace 3) Handle \n in output. 4) Document that "Given xxx" is optional. 5) Reject any non-matching comment lines starting with "given" or "outputs" 6) Fix missed test in ccan/cast Signed-off-by: Rusty Russell --- ccan/array_size/_info | 2 +- ccan/autodata/_info | 4 +- ccan/cast/_info | 2 +- ccan/crc/_info | 2 +- ccan/io/_info | 2 +- ccan/jmap/_info | 6 +- ccan/objset/_info | 6 +- ccan/rfc822/_info | 8 +- ccan/siphash/_info | 2 +- ccan/siphash/siphash.h | 2 +- ccan/strmap/_info | 6 +- ccan/strset/_info | 4 +- ccan/take/_info | 2 +- ccan/tal/link/_info | 4 +- ccan/tal/str/str.h | 6 +- ccan/tcon/_info | 4 +- doc/ccanlint.1 | 34 ++--- doc/ccanlint.1.txt | 12 +- tools/ccanlint/tests/examples_run.c | 186 ++++++++++++---------------- 19 files changed, 130 insertions(+), 164 deletions(-) diff --git a/ccan/array_size/_info b/ccan/array_size/_info index b8a9a850..69570f34 100644 --- a/ccan/array_size/_info +++ b/ccan/array_size/_info @@ -12,7 +12,7 @@ * macro or constant. * * Example: - * // Outputs "Initialized 32 values" + * // Outputs "Initialized 32 values\n" * #include * #include * #include diff --git a/ccan/autodata/_info b/ccan/autodata/_info index a101276d..507de1a1 100644 --- a/ccan/autodata/_info +++ b/ccan/autodata/_info @@ -92,8 +92,8 @@ * printf("verbose mode on\n"); * return 0; * } - * // Given -v outputs 'verbose mode on' - * // Given -v -C / outputs 'chdir to /. verbose mode on' + * // Given "-v" outputs "verbose mode on\n" + * // Given "-v -C /" outputs "chdir to /. verbose mode on\n" */ int main(int argc, char *argv[]) { diff --git a/ccan/cast/_info b/ccan/cast/_info index f5cd37fc..b09e0344 100644 --- a/ccan/cast/_info +++ b/ccan/cast/_info @@ -20,7 +20,7 @@ * License: LGPL (v2.1 or any later version) * * Example: - * // Given "test" contains "3 t's in 'test string' + * // Given "test" output contains "3 t's in 'test string'" * #include * #include * #include diff --git a/ccan/crc/_info b/ccan/crc/_info index c59d58ec..8a47db96 100644 --- a/ccan/crc/_info +++ b/ccan/crc/_info @@ -16,7 +16,7 @@ * #include * #include * - * // Given IHATEMATH outputs 0x98a3b8df + * // Given "IHATEMATH" outputs 0x98a3b8df * int main(int argc, char *argv[]) * { * if (argc != 2) { diff --git a/ccan/io/_info b/ccan/io/_info index 42b71d9e..fe49df5c 100644 --- a/ccan/io/_info +++ b/ccan/io/_info @@ -11,7 +11,7 @@ * plans. * * Example: - * // Given tr A-Z a-z outputs tr a-z a-z + * // Given "tr A-Z a-z" outputs tr a-z a-z * #include * #include * #include diff --git a/ccan/jmap/_info b/ccan/jmap/_info index 739e60af..ed1947fd 100644 --- a/ccan/jmap/_info +++ b/ccan/jmap/_info @@ -80,9 +80,9 @@ * jmap_free(arg); * return 0; * } - * // Given "--help" output contains "Arg 1 ('--help') is a long opt of 4 chars" - * // Given "-h" output contains "Arg 1 ('-h') is a short opt of 1 chars" - * // Given "foo" output contains "Arg 1 ('foo') is a normal arg of 3 chars" + * // Given "--help" output contains "Arg 1 ('--help') is a long opt of 4 chars\n" + * // Given "-h" output contains "Arg 1 ('-h') is a short opt of 1 chars\n" + * // Given "foo" output contains "Arg 1 ('foo') is a normal arg of 3 chars\n" * * License: LGPL (v2.1 or any later version) * Author: Rusty Russell diff --git a/ccan/objset/_info b/ccan/objset/_info index 5831f584..967764e7 100644 --- a/ccan/objset/_info +++ b/ccan/objset/_info @@ -41,9 +41,9 @@ * } * return 0; * } - * // Given 'a b c' outputs No arguments start with -. - * // Given 'a -b c' outputs 2, - * // Given 'a -b -c d' outputs 2,3, + * // Given "a b c" outputs No arguments start with -. + * // Given "a -b c" outputs 2, + * // Given "a -b -c d" outputs 2,3, */ int main(int argc, char *argv[]) { diff --git a/ccan/rfc822/_info b/ccan/rfc822/_info index 680f4991..5a0a9c6a 100644 --- a/ccan/rfc822/_info +++ b/ccan/rfc822/_info @@ -25,9 +25,9 @@ * it too). * * Example: - * // Given '' outputs 'body' - * // Given 'From' outputs ' ' - * // Given 'To' outputs ' ' + * // Outputs "body\n" + * // Given "From" outputs + * // Given "To" outputs * char buf[] = "From: \n" * "To: \n\n" * "body\n"; @@ -35,7 +35,7 @@ * struct bytestring out; * * msg = rfc822_start(NULL, buf, sizeof(buf)); - * if (!argv[1] || !argv[1][0]) + * if (!argv[1]) * out = rfc822_body(msg); * else { * struct rfc822_header *hdr; diff --git a/ccan/siphash/_info b/ccan/siphash/_info index d42ddab4..7aa31716 100644 --- a/ccan/siphash/_info +++ b/ccan/siphash/_info @@ -23,7 +23,7 @@ * Returns one 64-bit word as the hash function result. * * Example: - * // Outputs "cf2794e0277187b7" + * // Outputs cf2794e0277187b7 * #include * #include * diff --git a/ccan/siphash/siphash.h b/ccan/siphash/siphash.h index 55a391d2..6f39f9d5 100644 --- a/ccan/siphash/siphash.h +++ b/ccan/siphash/siphash.h @@ -28,7 +28,7 @@ * Returns one 64-bit word as the hash function result. * * Example: - * // Outputs "cf2794e0277187b7" + * // Outputs cf2794e0277187b7 * #include * #include * diff --git a/ccan/strmap/_info b/ccan/strmap/_info index 33176b30..55319a82 100644 --- a/ccan/strmap/_info +++ b/ccan/strmap/_info @@ -40,9 +40,9 @@ * printf("\n"); * return 0; * } - * // Given 'foo' outputs 'foo at 1. ' - * // Given 'foo bar' outputs 'bar at 2. foo at 1. ' - * // Given 'foo foo bar zebra' outputs 'bar at 3. foo at 1. zebra at 4. ' + * // Given "foo" outputs "foo at 1. \n" + * // Given "foo bar" outputs "bar at 2. foo at 1. \n" + * // Given "foo foo bar zebra" outputs "bar at 3. foo at 1. zebra at 4. \n" */ int main(int argc, char *argv[]) { diff --git a/ccan/strset/_info b/ccan/strset/_info index 3f4f7733..982a9c0f 100644 --- a/ccan/strset/_info +++ b/ccan/strset/_info @@ -45,8 +45,8 @@ * printf("\n"); * return 0; * } - * // Given "foo bar" outputs "bar foo " - * // Given "foo foo bar" outputs "bar foo " + * // Given "foo bar" outputs "bar foo \n" + * // Given "foo foo bar" outputs "bar foo \n" * * License: CC0 (but some dependencies are LGPL!) * Author: Rusty Russell diff --git a/ccan/take/_info b/ccan/take/_info index a3ac5392..69382466 100644 --- a/ccan/take/_info +++ b/ccan/take/_info @@ -13,7 +13,7 @@ * License: CC0 (Public domain) * * Example: - * // Given foo/bar.c outputs basename is bar.c + * // Given "foo/bar.c" outputs basename is bar.c * #include * #include * diff --git a/ccan/tal/link/_info b/ccan/tal/link/_info index 6962cb45..db2ad614 100644 --- a/ccan/tal/link/_info +++ b/ccan/tal/link/_info @@ -14,8 +14,8 @@ * // Silly program which keeps a cache of uppercased strings. * // The cache wants to keep strings around even after they may have * // been "freed" by the caller. - * // Given 'hello' outputs '1 cache hits HELLO ' - * // Given 'hello hello there' outputs '4 cache hits HELLO HELLO THERE ' + * // Given "hello" outputs "1 cache hits HELLO \n" + * // Given "hello hello there" outputs "4 cache hits HELLO HELLO THERE \n" * #include * #include * #include diff --git a/ccan/tal/str/str.h b/ccan/tal/str/str.h index 0fe542cf..0c182133 100644 --- a/ccan/tal/str/str.h +++ b/ccan/tal/str/str.h @@ -164,9 +164,9 @@ char *tal_strjoin(const void *ctx, char *strings[], const char *delim, * regcomp(3), regex(3). * * Example: - * // Given 'My name is Rusty' outputs 'Hello Rusty!' - * // Given 'my first name is Rusty Russell' outputs 'Hello Rusty Russell!' - * // Given 'My name isnt Rusty Russell' outputs 'Hello there!' + * // Given "My name is Rusty" outputs "Hello Rusty!\n" + * // Given "my first name is Rusty Russell" outputs "Hello Rusty Russell!\n" + * // Given "My name isnt Rusty Russell" outputs "Hello there!\n" * int main(int argc, char *argv[]) * { * char *person, *input; diff --git a/ccan/tcon/_info b/ccan/tcon/_info index c07e41ed..f6a6f0f9 100644 --- a/ccan/tcon/_info +++ b/ccan/tcon/_info @@ -54,8 +54,8 @@ * container_get(&sc), *container_get(&ic) - 1); * return 0; * } - * // Given "foo" outputs "Last arg is foo of 1 arguments" - * // Given "foo bar" outputs "Last arg is bar of 2 arguments" + * // Given "foo" outputs "Last arg is foo of 1 arguments\n" + * // Given "foo bar" outputs "Last arg is bar of 2 arguments\n" * * License: CC0 (Public domain) * diff --git a/doc/ccanlint.1 b/doc/ccanlint.1 index 4cc06c27..15260649 100644 --- a/doc/ccanlint.1 +++ b/doc/ccanlint.1 @@ -1,13 +1,13 @@ '\" t .\" Title: ccanlint .\" Author: [see the "AUTHOR" section] -.\" Generator: DocBook XSL Stylesheets v1.75.2 -.\" Date: 12/05/2011 +.\" Generator: DocBook XSL Stylesheets v1.78.1 +.\" Date: 09/28/2015 .\" Manual: \ \& .\" Source: \ \& .\" Language: English .\" -.TH "CCANLINT" "1" "12/05/2011" "\ \&" "\ \&" +.TH "CCANLINT" "1" "09/28/2015" "\ \&" "\ \&" .\" ----------------------------------------------------------------- .\" * Define some portability stuff .\" ----------------------------------------------------------------- @@ -36,7 +36,7 @@ ccanlint \- Make CCAN code modules, and the brightness up\&. .sp No encoder? No need to \fBccanlint\fR\&. You programmer? Excited to \fBccanlint\fR! .sp -CCAN module is small code of the song\&. \fBccanlint\fR full CCAN testing tool\&. Each test spray bit of wisdom\&. Also score\&. Good score good\&. Bad bad score\&. +CCAN module is small code of the song\&. \fBccanlint\fR full CCAN testing tool\&. Each test spray bit of wisdom\&. Also score\&. Good score good\&. Bad score bad\&. .sp \fBccanlint\fR expect the source code in this directory, or command line can be more than one\&. Exit 0 happy if all modules all tests happy\&. .SH "OPTIONS" @@ -73,14 +73,12 @@ Graphviz, then die happy\&. .PP \fB\-k, \-\-keep\fR .RS 4 - \fBccanlint\fR normally make mess temporary directory, but now it later in forensic\&. .RE .PP \fB\-s, \-\-summary\fR .RS 4 - \fBccanlint\fR just realized there is no message unless you die horrible\&. .RE @@ -107,7 +105,6 @@ Do not run all tests\&. Run this test, and the proof you need\&. Used many times .PP \fB\-\-compiler\fR=\fICOMPILER\fR .RS 4 - \fBccanlint\fR read config\&.h about finding \fICCAN_COMPILER\fR\&. Otherwise use the default when it was built\&. The change, to use this compiler\&. @@ -148,7 +145,6 @@ question may help to write one\&. .PP \fBdepends_exist\fR .RS 4 - \fI_info\fR file CCAN other module without saying, must find\&. It is not score 0\&. .RE @@ -197,8 +193,7 @@ unhappy\&. \fBhash_if\fR .RS 4 Module wants -\fBccanlint\fR -\fIconfig\&.h\fR +\fBccanlint\fR\fIconfig\&.h\fR "#define HAVE_FEATURE" for all feature\&. Function test "#if HAVE_FEATURE" no "#ifdef HAVE_FEATURE" because user might not know about the role at all\&. Intelligent GCC flag \fI\-Wundef\fR say HAVE_FEATURE not 0, not 1! but only if the use of @@ -207,7 +202,6 @@ say HAVE_FEATURE not 0, not 1! but only if the use of .PP \fBinfo_documentation_exists\fR .RS 4 - \fI_info\fR file format is pretty comments\&. Copying someone\&. It is not difficult write documentation! .RE @@ -250,7 +244,6 @@ Hostile to BSD license module, but requires another module of the GPL\&. Perhaps .PP \fBmain_header_exists\fR .RS 4 - \fBccanlint\fR know the module name directory name\&. Expect the same name for header\&. .RE @@ -283,7 +276,6 @@ Linux kernel programmers more, solve the problem for the space of the final ban .PP \fBexamples_compile\fR .RS 4 - \fBccanlint\fR very smart! Take \fIExample:\fR @@ -302,15 +294,16 @@ says wow! \fBexamples_run\fR .RS 4 If the example program that comments like -\fI// given foo outputs bar\fR -\fBccanlint\fR -will run the food program +\fI// Given "foo" outputs "bar"\fR\fBccanlint\fR +will run the program with \fIfoo\fR in the command line and standard input\&. Happy if \fIbar\fR -are out\&. You can do \*(Aq or " about the production or determined\&. You can also -\fIoutput contains\fR -more relaxed controls\&. +are out\&. If quotes around +\fIbar\fR +exact match needed; without quotes whitespace matches any other space and trailing ignored\&. \en is also supported for matching\&. You can also +\fI"output contains"\fR +to pass if the output contains the string\&. .RE .PP \fBmodule_links\fR @@ -364,7 +357,6 @@ Other files .PP \fBtests_pass\fR .RS 4 - \fIrun\fR and \fIapi\fR @@ -373,7 +365,6 @@ test happy departure\&. If not happy, offer debugger\&. .PP \fBtests_pass_valgrind\fR .RS 4 - \fBvalgrind\fR the tool of all \fIrun\fR @@ -390,7 +381,6 @@ section, make "tests_pass_valgrind test/TESTNAME:FAIL"\&. If required valgrind a .PP \fBtests_pass_valgrind_noleaks\fR .RS 4 - \fBvalgrind\fR complain if the memory leak test\&. \fI_info\fR diff --git a/doc/ccanlint.1.txt b/doc/ccanlint.1.txt index ea321809..26d46f4c 100644 --- a/doc/ccanlint.1.txt +++ b/doc/ccanlint.1.txt @@ -175,11 +175,13 @@ test, but happy: bad example *ccanlint* says wow! *examples_run*:: - If the example program that comments like '// given foo outputs bar' - *ccanlint* will run the food program 'foo' in the command line and - standard input. Happy if 'bar' are out. You can do ' or " about - the production or determined. You can also 'output contains' more - relaxed controls. + If the example program that comments like '// [Given "foo"] outputs + "bar"' then *ccanlint* will run the program with 'foo' in the + command line and standard input. Happy if 'bar' are out and exit 0. + If quotes around 'bar' exact match needed; without quotes whitespace matches + any other space and trailing ignored. \n is also supported for + matching. You can also '"output contains"' to pass if the output + contains the string. *module_links*:: CCAN link to the program module simply no error. diff --git a/tools/ccanlint/tests/examples_run.c b/tools/ccanlint/tests/examples_run.c index 0e473ca4..3bdb655e 100644 --- a/tools/ccanlint/tests/examples_run.c +++ b/tools/ccanlint/tests/examples_run.c @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -81,146 +82,108 @@ static bool scan_for(const void *ctx, const char *input, const char *fmt, ...) } static char *find_expect(struct ccan_file *file, - char **lines, char **input, bool *exact, + char **lines, char **input, + bool *contains, bool *whitespace, bool *error, unsigned *line) { - char *expect; - const char *fmt; + char *rest, *expect; + *error = false; for (; lines[*line]; (*line)++) { char *p = lines[*line] + strspn(lines[*line], " \t"); if (!strstarts(p, "//")) continue; p += strspn(p, "/ "); - foreach_ptr(fmt, - "given '%s', outputs '%s'", - "given '%s' outputs '%s'", - "given \"%s\", outputs \"%s\"", - "given \"%s\" outputs \"%s\"") { - if (scan_for(file, p, fmt, input, &expect)) { - *exact = true; - return expect; - } - } - - foreach_ptr(fmt, - "given '%s', output contains '%s'", - "given '%s' output contains '%s'", - "given \"%s\", output contains \"%s\"", - "given \"%s\" output contains \"%s\"") { - if (scan_for(file, p, fmt, input, &expect)) { - *exact = false; - return expect; - } - } - foreach_ptr(fmt, "outputs '%s'", "outputs \"%s\"") { - if (scan_for(file, p, fmt, &expect)) { - *input = cast_const(char *, ""); - *exact = true; - return expect; + /* With or without input? */ + if (strncasecmp(p, "given", strlen("given")) == 0) { + /* Must be of form */ + if (!scan_for(file, p, "given \"%s\" %s", input, &p)) { + *error = true; + return p; } + } else { + *input = NULL; } - foreach_ptr(fmt, - "given '%s', output contains '%s'", - "given '%s' output contains '%s'", - "given \"%s\", output contains \"%s\"", - "given \"%s\" output contains \"%s\"") { - if (scan_for(file, p, fmt, input, &expect)) { - *exact = false; - return expect; - } + if (scan_for(file, p, "outputs \"%s\"", &expect)) { + *whitespace = true; + *contains = false; + return expect; } - /* Unquoted versions... we can get this wrong! */ - foreach_ptr(fmt, - "given %s, outputs '%s'", - "given '%s', outputs %s", - "given %s, outputs \"%s\"", - "given \"%s\", outputs %s", - "given %s, outputs %s", - "given %s outputs '%s'", - "given '%s' outputs %s", - "given %s outputs \"%s\"", - "given \"%s\" outputs %s", - "given %s outputs %s") { - if (scan_for(file, p, fmt, input, &expect)) { - *exact = true; - return expect; - } + if (scan_for(file, p, "output contains \"%s\"", &expect)) { + *whitespace = true; + *contains = true; + return expect; } - foreach_ptr(fmt, - "given %s, output contains '%s'", - "given '%s', output contains %s", - "given %s, output contains \"%s\"", - "given \"%s\", output contains %s", - "given %s, output contains %s", - "given %s output contains '%s'", - "given '%s' output contains %s", - "given %s output contains \"%s\"", - "given \"%s\" output contains %s", - "given %s output contains %s") { - if (scan_for(file, p, fmt, input, &expect)) { - *exact = false; - return expect; - } + /* Whitespace-ignoring versions. */ + if (scan_for(file, p, "outputs %s", &expect)) { + *whitespace = false; + *contains = false; + return expect; } - foreach_ptr(fmt, - "outputs '%s'", - "outputs \"%s\"", - "outputs %s") { - if (scan_for(file, p, fmt, &expect)) { - *input = cast_const(char *, ""); - *exact = true; - return expect; - } + if (scan_for(file, p, "output contains %s", &expect)) { + *whitespace = false; + *contains = true; + return expect; } - foreach_ptr(fmt, - "output contains '%s'", - "output contains \"%s\"", - "output contains %s") { - if (scan_for(file, p, fmt, &expect)) { - *input = cast_const(char *, ""); - *exact = false; - return expect; - } + /* Other malformed line? */ + if (*input || !strncasecmp(p, "output", strlen("output"))) { + *error = true; + return p; } - } + } return NULL; } -static char *trim(char *string) -{ - while (strends(string, "\n")) - string[strlen(string)-1] = '\0'; - return string; -} - static char *unexpected(struct ccan_file *i, const char *input, - const char *expect, bool exact) + const char *expect, bool contains, bool whitespace) { char *output, *cmd; + const char *p; bool ok; unsigned int default_time = default_timeout_ms; - cmd = tal_fmt(i, "echo '%s' | %s %s", - input, i->compiled[COMPILE_NORMAL], input); + if (input) + cmd = tal_fmt(i, "echo '%s' | %s %s", + input, i->compiled[COMPILE_NORMAL], input); + else + cmd = tal_fmt(i, "%s", i->compiled[COMPILE_NORMAL]); output = run_with_timeout(i, cmd, &ok, &default_time); if (!ok) return tal_fmt(i, "Exited with non-zero status\n"); - if (exact) { - if (streq(output, expect) || streq(trim(output), expect)) + /* Substitute \n */ + while ((p = strstr(expect, "\\n")) != NULL) { + expect = tal_fmt(cmd, "%.*s\n%s", (int)(p - expect), expect, + p+2); + } + + if (!whitespace) { + /* Normalize to spaces. */ + expect = tal_strjoin(cmd, + tal_strsplit(cmd, expect, " \n\t", + STR_NO_EMPTY), + " ", STR_NO_TRAIL); + output = tal_strjoin(cmd, + tal_strsplit(cmd, output, " \n\t", + STR_NO_EMPTY), + " ", STR_NO_TRAIL); + } + + if (contains) { + if (strstr(output, expect)) return NULL; } else { - if (strstr(output, expect)) + if (streq(output, expect)) return NULL; } + return output; } @@ -237,21 +200,32 @@ static void run_examples(struct manifest *m, list_for_each(list, i, list) { char **lines, *expect, *input, *output; unsigned int linenum = 0; - bool exact; + bool contains, whitespace, error; lines = get_ccan_file_lines(i); - for (expect = find_expect(i, lines, &input, &exact, + for (expect = find_expect(i, lines, &input, + &contains, &whitespace, &error, &linenum); expect; linenum++, expect = find_expect(i, lines, &input, - &exact, &linenum)) { + &contains, &whitespace, + &error, &linenum)) { + if (error) { + score_file_error(score, i, linenum+1, + "Unparsable test line '%s'", + lines[linenum]); + score->pass = false; + break; + } + if (i->compiled[COMPILE_NORMAL] == NULL) continue; score->total++; - output = unexpected(i, input, expect, exact); + output = unexpected(i, input, expect, + contains, whitespace); if (!output) { score->score++; continue; @@ -259,7 +233,7 @@ static void run_examples(struct manifest *m, score_file_error(score, i, linenum+1, "output '%s' didn't %s '%s'\n", output, - exact ? "match" : "contain", + contains ? "contain" : "match", expect); score->pass = false; } -- 2.39.2