6 * ccan_tokenizer - A full-text lexer for C source files
8 * ccan_tokenizer generates a list of tokens given the contents of a C source
13 * #include <ccan/ccan_tokenizer/ccan_tokenizer.h>
14 * #include <ccan/grab_file/grab_file.h>
17 * static void token_list_stats(const struct token_list *tl) {
18 * size_t comment=0, white=0, stray=0, code=0, total=0;
20 * const struct token *i;
22 * for (i=tl->first; i; i=i->next) {
23 * size_t size = i->orig_size;
27 * if (token_type_is_comment(i->type))
29 * else if (i->type == TOK_WHITE)
31 * else if (i->type == TOK_STRAY)
37 * printf("Code: %.02f%%\n"
38 * "White space: %.02f%%\n"
39 * "Comments: %.02f%%\n",
40 * (double)code * 100.0 / (double)total,
41 * (double)white * 100.0 / (double)total,
42 * (double)comment * 100.0 / (double)total);
44 * printf("Stray: %.02f%%\n",
45 * (double)stray * 100.0 / (double)total);
46 * printf("Total size: %zu bytes with %zu tokens\n",
50 * int main(int argc, char *argv[]) {
53 * struct token_list *tl;
54 * tok_message_queue mq;
55 * queue_init(mq, NULL);
59 * fprintf(stderr, "Usage: %s source_file\n", argv[0]);
62 * file = grab_file(NULL, argv[1], &len);
64 * err(1, "Could not read file %s", argv[1]);
66 * //tokenize the contents
67 * tl = tokenize(file, file, len, &mq);
69 * //print warnings, errors, etc.
70 * while (queue_count(mq)) {
71 * struct tok_message msg = dequeue(mq);
72 * tok_message_print(&msg, tl);
75 * //do neat stuff with the token list
76 * token_list_stats(tl);
79 * talloc_free(file); //implicitly frees tl
87 int main(int argc, char *argv[])
89 /* Expect exactly one argument */
93 if (strcmp(argv[1], "depends") == 0) {
94 printf("ccan/darray\n");
95 printf("ccan/talloc\n");