1 #include <ccan/ccan_tokenizer/read_cnumber.c>
2 #include <ccan/ccan_tokenizer/read_cstring.c>
3 #include <ccan/ccan_tokenizer/dict.c>
4 #include <ccan/ccan_tokenizer/ccan_tokenizer.c>
5 #include <ccan/ccan_tokenizer/queue.c>
6 #include <ccan/ccan_tokenizer/charflag.c>
7 #include <ccan/tap/tap.h>
9 #define item(num) (toks->first[num])
10 //sed 's/toks->array\.item\[\([^]]*\)\]/item(\1)/g'
12 tok_message_queue *MQ = NULL;
14 static const char *onechar_tokens = "!~#%^&*()=-+{}[]|;:,.<>/?";
15 static const char *twochar_tokens = "!=##%=^=&=&&*=-=--->+=++==|=||<=<<>=>>/=";
16 static const char *threechar_tokens = "<<=>>=...";
17 static const char *char_token = "'x'";
18 static const char *string_token = "\"string\"";
19 static const char *ident_tokens = "doh abc f_o _ba b$f";
21 static char *backslashify(const char *string)
24 char *ret = talloc_size(NULL, strlen(string)*3 + 1);
25 for (i = 0; i < strlen(string); i++) {
34 static char *spacify(const char *string, unsigned int num)
37 char *ret = talloc_size(NULL, strlen(string)*2 + 1);
38 memset(ret, ' ', strlen(string)*2);
40 for (i = 0; i < strlen(string); i += num)
41 memcpy(&ret[i + i/num], string+i, num);
42 ret[i + i/num] = '\0';
46 static struct token_list *test_tokens(const char *orig, unsigned int size)
48 struct token_list *toks;
49 char *string = talloc_strdup(NULL, orig);
52 toks = tokenize(string, string, strlen(string), MQ);
53 ok1(token_list_sanity_check(toks, stdout));
55 ok1(token_list_count(toks) == strlen(string)/size + 1);
56 ok1(item(0).type == TOK_STARTLINE);
57 for (i = 0; i < strlen(string)/size; i++) {
58 ok1(item(i+1).type == TOK_OPERATOR);
59 ok1(item(i+1).txt_size == size);
60 ok1(strncmp(item(i+1).txt, string + i*size, size) == 0);
61 ok1(item(i+1).orig_size == size);
62 ok1(item(i+1).orig == string + i*size);
67 static struct token_list *test_tokens_spaced(const char *orig,
70 struct token_list *toks;
71 char *string = spacify(orig, size);
74 toks = tokenize(string, string, strlen(string), MQ);
75 ok1(token_list_sanity_check(toks, stdout));
77 ok1(token_list_count(toks) == strlen(orig)/size*2 + 1);
78 ok1(item(0).type == TOK_STARTLINE);
79 for (i = 0; i < strlen(orig)/size; i++) {
80 ok1(item(i*2+1).type == TOK_OPERATOR);
81 ok1(item(i*2+1).txt_size == size);
82 ok1(!strncmp(item(i*2+1).txt, string + i*(size+1), size));
83 ok1(item(i*2+1).orig_size == size);
84 ok1(item(i*2+1).orig == string + i*(size+1));
85 ok1(item(i*2+2).type == TOK_WHITE);
86 ok1(item(i*2+2).txt_size == 1);
87 ok1(item(i*2+2).txt[0] == ' ');
88 ok1(item(i*2+2).orig_size == 1);
89 ok1(item(i*2+2).orig == string + i*(size+1) + size);
94 static struct token_list *test_tokens_backslashed(const char *orig,
97 struct token_list *toks;
98 const char *string = backslashify(orig);
101 toks = tokenize(string, string, strlen(string), MQ);
102 ok1(token_list_sanity_check(toks, stdout));
104 ok1(token_list_count(toks) == strlen(orig)/size + 1);
105 ok1(item(0).type == TOK_STARTLINE);
106 for (i = 0; i < strlen(orig)/size; i++) {
107 ok1(item(i+1).type == TOK_OPERATOR);
108 ok1(item(i+1).txt_size == size);
109 ok1(strncmp(item(i+1).txt, orig + i*size, size) == 0);
110 ok1(item(i+1).orig_size == size*3);
111 ok1(item(i+1).orig == string + i*size*3);
116 static void onechar_checks(const struct token_list *toks, int mul)
119 for (i = 0; i < strlen(onechar_tokens); i++)
120 ok1(item(i*mul+1).opkw == onechar_tokens[i]);
123 static void twochar_checks(const struct token_list *toks, int mul)
125 ok1(item(1).opkw == NE_OP);
126 ok1(item(1*mul+1).opkw == DOUBLE_POUND);
127 ok1(item(2*mul+1).opkw == MOD_ASSIGN);
128 ok1(item(3*mul+1).opkw == XOR_ASSIGN);
129 ok1(item(4*mul+1).opkw == AND_ASSIGN);
130 ok1(item(5*mul+1).opkw == AND_OP);
131 ok1(item(6*mul+1).opkw == MUL_ASSIGN);
132 ok1(item(7*mul+1).opkw == SUB_ASSIGN);
133 ok1(item(8*mul+1).opkw == DEC_OP);
134 ok1(item(9*mul+1).opkw == PTR_OP);
135 ok1(item(10*mul+1).opkw == ADD_ASSIGN);
136 ok1(item(11*mul+1).opkw == INC_OP);
137 ok1(item(12*mul+1).opkw == EQ_OP);
138 ok1(item(13*mul+1).opkw == OR_ASSIGN);
139 ok1(item(14*mul+1).opkw == OR_OP);
140 ok1(item(15*mul+1).opkw == LE_OP);
141 ok1(item(16*mul+1).opkw == LEFT_OP);
142 ok1(item(17*mul+1).opkw == GE_OP);
143 ok1(item(18*mul+1).opkw == RIGHT_OP);
144 ok1(item(19*mul+1).opkw == DIV_ASSIGN);
147 static void threechar_checks(const struct token_list *toks, int mul)
149 ok1(item(1).opkw == LEFT_ASSIGN);
150 ok1(item(1*mul+1).opkw == RIGHT_ASSIGN);
151 ok1(item(2*mul+1).opkw == ELLIPSIS);
157 struct token_list *toks;
159 char *backslashed_idents;
162 toks = test_tokens(onechar_tokens, 1);
163 onechar_checks(toks, 1);
164 talloc_free((char*)toks->orig);
166 toks = test_tokens(twochar_tokens, 2);
167 twochar_checks(toks, 1);
168 talloc_free((char*)toks->orig);
170 toks = test_tokens(threechar_tokens, 3);
171 threechar_checks(toks, 1);
172 talloc_free((char*)toks->orig);
175 str = talloc_strdup(NULL, char_token);
176 toks = tokenize(str, str, strlen(str), MQ);
177 ok1(token_list_sanity_check(toks, stdout));
178 ok1(token_list_count(toks) == 2);
179 ok1(item(0).type == TOK_STARTLINE);
180 ok1(item(1).type == TOK_CHAR);
181 ok1(item(1).txt_size == strlen(str));
182 ok1(strncmp(item(1).txt, str, strlen(str)) == 0);
183 ok1(item(1).orig_size == strlen(str));
184 ok1(item(1).orig == str);
185 /* FIXME: test contents of string. */
189 str = talloc_strdup(NULL, string_token);
190 toks = tokenize(str, str, strlen(str), MQ);
191 ok1(token_list_sanity_check(toks, stdout));
192 ok1(token_list_count(toks) == 2);
193 ok1(item(0).type == TOK_STARTLINE);
194 ok1(item(1).type == TOK_STRING);
195 ok1(item(1).txt_size == strlen(str));
196 ok1(strncmp(item(1).txt, str, strlen(str)) == 0);
197 ok1(item(1).orig_size == strlen(str));
198 ok1(item(1).orig == str);
199 /* FIXME: test contents of string. */
203 str = talloc_strdup(NULL, ident_tokens);
204 toks = tokenize(str, str, strlen(str), MQ);
205 ok1(token_list_sanity_check(toks, stdout));
206 token_list_dump(toks, stdout);
207 ok1(token_list_count(toks) == 10);
208 ok1(item(0).type == TOK_STARTLINE);
209 for (i = 0; i < 5; i++) {
210 ok1(item(i*2+1).type == TOK_IDENTIFIER);
211 ok1(item(i*2+1).txt_size == 3);
212 ok1(strncmp(item(i*2+1).txt, str + i*4, 3) == 0);
213 ok1(item(i*2+1).orig_size == 3);
214 ok1(item(i*2+1).orig == str + i*4);
217 ok1(item(i*2+2).type == TOK_WHITE);
218 ok1(item(i*2+2).txt_size == 1);
219 ok1(item(i*2+2).txt[0] == ' ');
220 ok1(item(i*2+2).orig_size == 1);
221 ok1(item(i*2+2).orig == str + i*4 + 3);
225 toks = test_tokens_spaced(onechar_tokens, 1);
226 onechar_checks(toks, 2);
227 talloc_free((char*)toks->orig);
229 toks = test_tokens_spaced(twochar_tokens, 2);
230 twochar_checks(toks, 2);
231 talloc_free((char*)toks->orig);
233 toks = test_tokens_spaced(threechar_tokens, 3);
234 threechar_checks(toks, 2);
235 talloc_free((char*)toks->orig);
237 toks = test_tokens_backslashed(onechar_tokens, 1);
238 onechar_checks(toks, 1);
239 talloc_free((char*)toks->orig);
241 toks = test_tokens_backslashed(twochar_tokens, 2);
242 twochar_checks(toks, 1);
243 talloc_free((char*)toks->orig);
245 toks = test_tokens_backslashed(threechar_tokens, 3);
246 threechar_checks(toks, 1);
247 talloc_free((char*)toks->orig);
250 backslashed_idents = backslashify(ident_tokens);
251 toks = tokenize(backslashed_idents, backslashed_idents, strlen(backslashed_idents), MQ);
252 ok1(token_list_sanity_check(toks, stdout));
253 ok1(token_list_count(toks) == 10);
254 ok1(item(0).type == TOK_STARTLINE);
255 for (i = 0; i < 5; i++) {
256 ok1(item(i*2+1).type == TOK_IDENTIFIER);
257 ok1(item(i*2+1).txt_size == 3);
258 ok1(strncmp(item(i*2+1).txt, ident_tokens + i*4, 3) == 0);
259 ok1(item(i*2+1).orig_size == 9);
260 ok1(item(i*2+1).orig == backslashed_idents + i*12);
263 ok1(item(i*2+2).type == TOK_WHITE);
264 ok1(item(i*2+2).txt_size == 1);
265 ok1(item(i*2+2).txt[0] == ' ');
266 ok1(item(i*2+2).orig_size == 3);
267 ok1(item(i*2+2).orig == backslashed_idents + i*12 + 9);
269 talloc_free(backslashed_idents);
271 return exit_status();