1 /* MIT (BSD) license - see LICENSE file for details */
4 #include "ccan/tal/str/str.h"
5 #include "ccan/utf8/utf8.h"
8 // GraphQL character classes
10 #define SOURCE_CHAR(c) ((c) == 9 || (c) == 10 || (c) == 13 || ((c) >= 32 && (c) <= 65535))
11 #define WHITE_SPACE(c) ((c) == 9 || (c) == 32)
12 #define LINE_TERMINATOR(c) ((c) == 10 || (c) == 13)
13 #define COMMENT(c) ((c) == 35)
14 #define COMMENT_CHAR(c) (SOURCE_CHAR(c) && !LINE_TERMINATOR(c))
15 #define STRING_CHAR(c) (SOURCE_CHAR(c) && !LINE_TERMINATOR(c) && (c)!='"' && (c)!='\\')
16 #define BLOCK_STRING_CHAR(c) (SOURCE_CHAR(c))
17 #define COMMA(c) ((c) == 44)
18 #define EOF_CHAR(c) ((c) == 0 || (c) == 4)
19 #define PUNCTUATOR(c) (strchr("!$&().:=@[]{|}", c))
20 #define HEX_DIGIT(c) (DIGIT(c) || ((c) >= 0x61 && (c) <= 0x66) || ((c) >= 0x41 && (c) <= 0x46))
21 #define DIGIT(c) ((c) >= 0x30 && (c) <= 0x39)
22 #define NAME_START(c) (((c) >= 0x61 && (c) <= 0x7A) || ((c) >= 0x41 && (c) <= 0x5A) || (c) == 0x5F)
23 #define NAME_CONTINUE(c) (NAME_START(c) || DIGIT(c))
26 #define cpystr(d,s) { char *cpystr_p; char *cpystr_q; for(cpystr_p = (s), cpystr_q = (d); *cpystr_p;) *cpystr_q++ = *cpystr_p++; *cpystr_q++ = *cpystr_p++; }
31 #define PARAMS struct list_head *tokens, struct list_head *used, const char **err
32 #define ARGS tokens, used, err
34 struct graphql_token *rollback_top = list_top(tokens, struct graphql_token, list); \
35 struct graphql_##type *obj = tal(tokens, struct graphql_##type); memset(obj, 0, sizeof(struct graphql_##type)); \
40 rollback_top = rollback_top; \
41 if (*err) obj = tal_free(obj); \
44 #define CONSUME_ONE { list_add(used, (struct list_node *)list_pop(tokens, struct graphql_token, list)); }
45 #define RESTORE_ONE { list_add(tokens, (struct list_node *)list_pop(used, struct graphql_token, list)); }
46 #define ROLLBACK(args) { while (list_top(tokens, struct graphql_token, list) != rollback_top) { RESTORE_ONE; } }
47 #define OR if (!*err) goto exit_label; *err = NULL;
48 #define REQ if (*err) { ROLLBACK(args); goto exit_label; }
49 #define OPT *err = NULL;
50 #define WHILE_OPT while(!*err); *err = NULL;
51 #define LOOKAHEAD(args, tok) struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
52 #define MSG(msg) if (*err) *err = msg;
57 RET parse_document(PARAMS);
58 RET parse_definition(PARAMS);
59 RET parse_executable_document(PARAMS);
60 RET parse_executable_definition(PARAMS);
61 RET parse_operation_definition(PARAMS);
62 RET parse_operation_type(PARAMS);
63 RET parse_selection_set(PARAMS);
64 RET parse_selection(PARAMS);
65 RET parse_field(PARAMS);
66 RET parse_arguments(PARAMS);
67 RET parse_argument(PARAMS);
68 RET parse_alias(PARAMS);
69 RET parse_fragment_spread(PARAMS);
70 RET parse_fragment_definition(PARAMS);
71 RET parse_fragment_name(PARAMS);
72 RET parse_type_condition(PARAMS);
73 RET parse_inline_fragment(PARAMS);
74 RET parse_value(PARAMS);
75 RET parse_int_value(PARAMS);
76 RET parse_negative_sign(PARAMS);
77 RET parse_non_zero_digit(PARAMS);
78 RET parse_float_value(PARAMS);
79 RET parse_boolean_value(PARAMS);
80 RET parse_string_value(PARAMS);
81 RET parse_string_character(PARAMS);
82 RET parse_escaped_unicode(PARAMS);
83 RET parse_escaped_character(PARAMS);
84 RET parse_block_string_character(PARAMS);
85 RET parse_null_value(PARAMS);
86 RET parse_enum_value(PARAMS);
87 RET parse_list_value(PARAMS);
88 RET parse_object_value(PARAMS);
89 RET parse_object_field(PARAMS);
90 RET parse_variable(PARAMS);
91 RET parse_variable_definitions(PARAMS);
92 RET parse_variable_definition(PARAMS);
93 RET parse_default_value(PARAMS);
94 RET parse_type(PARAMS);
95 RET parse_named_type(PARAMS);
96 RET parse_list_type(PARAMS);
97 RET parse_non_null_type(PARAMS);
98 RET parse_non_null_type_1(PARAMS);
99 RET parse_non_null_type_2(PARAMS);
100 RET parse_directives(PARAMS);
101 RET parse_directive(PARAMS);
102 RET parse_type_system_document(PARAMS);
103 RET parse_type_system_definition(PARAMS);
104 RET parse_type_system_extension_document(PARAMS);
105 RET parse_type_system_definition_or_extension(PARAMS);
106 RET parse_type_system_extension(PARAMS);
107 RET parse_description(PARAMS);
108 RET parse_schema_definition(PARAMS);
109 RET parse_root_operation_type_definition(PARAMS);
110 RET parse_schema_extension(PARAMS);
111 RET parse_type_definition(PARAMS);
112 RET parse_type_extension(PARAMS);
113 RET parse_scalar_type_definition(PARAMS);
114 RET parse_scalar_type_extension(PARAMS);
115 RET parse_object_type_definition(PARAMS);
116 RET parse_implements_interfaces(PARAMS);
117 RET parse_fields_definition(PARAMS);
118 RET parse_field_definition(PARAMS);
119 RET parse_arguments_definition(PARAMS);
120 RET parse_input_value_definition(PARAMS);
121 RET parse_object_type_extension(PARAMS);
122 RET parse_interface_type_definition(PARAMS);
123 RET parse_interface_type_extension(PARAMS);
124 RET parse_union_type_definition(PARAMS);
125 RET parse_union_member_types(PARAMS);
126 RET parse_union_type_extension(PARAMS);
127 RET parse_enum_type_definition(PARAMS);
128 RET parse_enum_values_definition(PARAMS);
129 RET parse_enum_value_definition(PARAMS);
130 RET parse_enum_type_extension(PARAMS);
131 RET parse_input_object_type_definition(PARAMS);
132 RET parse_input_fields_definition(PARAMS);
133 RET parse_directive_definition(PARAMS);
134 RET parse_directive_locations(PARAMS);
135 RET parse_directive_location(PARAMS);
136 RET parse_executable_directive_location(PARAMS);
137 RET parse_type_system_directive_location(PARAMS);
139 RET parse_keyword(PARAMS, const char *keyword, const char *errmsg);
140 RET parse_punct(PARAMS, int punct);
141 RET parse_name(PARAMS);
142 RET parse_int(PARAMS);
143 RET parse_float(PARAMS);
144 RET parse_string(PARAMS);
146 // Convert input string into AST.
147 const char *graphql_lexparse(const char *input, const tal_t *ctx, struct list_head **tokens, struct graphql_executable_document **doc) {
148 const char *err = graphql_lex(input, ctx, tokens);
150 err = graphql_parse(*tokens, doc);
154 // Convert lexed tokens into AST.
155 const char *graphql_parse(struct list_head *tokens, struct graphql_executable_document **doc) {
156 struct list_head used = LIST_HEAD_INIT(used);
157 const char *err = NULL;
158 *doc = parse_executable_document(tokens, &used, &err);
162 /* The following parser functions follow special rules:
163 * - The declaration is standardized with RET and PARAMS
164 * - The "err" argument is assumed to be NULL upon entrance
165 * - The "err" argument is set on failure
166 * - If the function fails to parse, then "tokens" shall be as it was upon entrance
167 * - INIT and EXIT macros are used
168 * - Macros such as REQ and OPT facilitate readability and conciseness
171 RET parse_document(PARAMS) {
173 obj->first_def = parse_definition(ARGS); REQ
174 struct graphql_definition *p = obj->first_def;
176 p->next_def = parse_definition(ARGS);
182 RET parse_definition(PARAMS) {
184 obj->executable_def = parse_executable_definition(ARGS);
186 obj->type_system_def = parse_type_system_definition_or_extension(ARGS);
187 // NOTE: Optional type system is not (yet) implemented.
192 RET parse_executable_document(PARAMS) {
193 INIT(executable_document);
194 obj->first_def = parse_executable_definition(ARGS); REQ
195 struct graphql_executable_definition *p = obj->first_def;
197 p->next_def = parse_executable_definition(ARGS);
203 RET parse_executable_definition(PARAMS) {
204 INIT(executable_definition);
205 obj->op_def = parse_operation_definition(ARGS); MSG("invalid operation or fragment definition"); OR
206 obj->frag_def = parse_fragment_definition(ARGS); MSG("invalid operation or fragment definition");
210 RET parse_operation_definition(PARAMS) {
211 INIT(operation_definition);
212 obj->op_type = parse_operation_type(ARGS);
214 obj->op_name = parse_name(ARGS); OPT
215 obj->vars = parse_variable_definitions(ARGS); OPT
216 obj->directives = parse_directives(ARGS); OPT
219 obj->sel_set = parse_selection_set(ARGS);
220 if (*err) ROLLBACK(ARGS);
224 RET parse_operation_type(PARAMS) {
225 INIT(operation_type);
226 const char *errmsg = "expected: query, mutation, or subscription";
227 obj->op_type = parse_keyword(ARGS, "query", errmsg); OR
228 obj->op_type = parse_keyword(ARGS, "mutation", errmsg); OR
229 obj->op_type = parse_keyword(ARGS, "subscription", errmsg);
233 RET parse_selection_set(PARAMS) {
235 parse_punct(ARGS, '{'); REQ;
236 obj->first = parse_selection(ARGS); REQ;
237 struct graphql_selection *p = obj->first;
238 parse_punct(ARGS, '}');
241 p->next = parse_selection(ARGS); MSG("expected: selection or '}'"); REQ;
243 parse_punct(ARGS, '}');
248 RET parse_selection(PARAMS) {
250 obj->field = parse_field(ARGS); OR
251 obj->frag_spread = parse_fragment_spread(ARGS); OR
252 obj->inline_frag = parse_inline_fragment(ARGS);
253 MSG("expected: field, fragment spread, or inline fragment");
257 RET parse_field(PARAMS) {
259 obj->alias = parse_alias(ARGS); OPT
260 obj->name = parse_name(ARGS); REQ
261 obj->args = parse_arguments(ARGS); OPT
262 obj->directives = parse_directives(ARGS); OPT
263 obj->sel_set = parse_selection_set(ARGS); OPT
267 RET parse_arguments(PARAMS) {
269 parse_punct(ARGS, '('); REQ
270 obj->first = parse_argument(ARGS); REQ
271 struct graphql_argument *p = obj->first;
272 parse_punct(ARGS, ')');
275 p->next = parse_argument(ARGS); MSG("expected: argument or ')'"); REQ;
277 parse_punct(ARGS, ')');
282 RET parse_argument(PARAMS) {
284 obj->name = parse_name(ARGS); REQ
285 parse_punct(ARGS, ':'); REQ
286 obj->val = parse_value(ARGS); REQ
290 RET parse_alias(PARAMS) {
292 obj->name = parse_name(ARGS); REQ
293 parse_punct(ARGS, ':'); REQ
297 RET parse_fragment_spread(PARAMS) {
298 INIT(fragment_spread);
299 parse_punct(ARGS, 0x2026); REQ // ...
300 obj->name = parse_fragment_name(ARGS); REQ
301 obj->directives = parse_directives(ARGS); OPT
305 RET parse_fragment_definition(PARAMS) {
306 INIT(fragment_definition);
307 parse_keyword(ARGS, "fragment", "fragment expected"); REQ
308 obj->name = parse_fragment_name(ARGS); REQ
309 obj->type_cond = parse_type_condition(ARGS); REQ
310 obj->directives = parse_directives(ARGS); OPT
311 obj->sel_set = parse_selection_set(ARGS); REQ
315 RET parse_fragment_name(PARAMS) {
317 obj->name = parse_name(ARGS); REQ
318 struct graphql_token *tok = list_top(used, struct graphql_token, list);
319 if (streq(tok->token_string, "on")) {
320 *err = "invalid fragment name";
326 RET parse_type_condition(PARAMS) {
327 INIT(type_condition);
328 parse_keyword(ARGS, "on", "expected: 'on'"); REQ
329 obj->named_type = parse_named_type(ARGS); REQ
333 RET parse_inline_fragment(PARAMS) {
334 INIT(inline_fragment);
335 parse_punct(ARGS, 0x2026); REQ // ...
336 obj->type_cond = parse_type_condition(ARGS); OPT
337 obj->directives = parse_directives(ARGS); OPT
338 obj->sel_set = parse_selection_set(ARGS); REQ
342 RET parse_value(PARAMS) {
344 obj->var = parse_variable(ARGS); // FIXME: if not const
346 obj->int_val = parse_int_value(ARGS); OR
347 obj->float_val = parse_float_value(ARGS); OR
348 obj->str_val = parse_string_value(ARGS); OR
349 obj->bool_val = parse_boolean_value(ARGS); OR
350 obj->null_val = parse_null_value(ARGS); OR
351 obj->enum_val = parse_enum_value(ARGS); OR
352 obj->list_val = parse_list_value(ARGS); OR
353 obj->obj_val = parse_object_value(ARGS);
357 RET parse_int_value(PARAMS) {
359 obj->val = parse_int(ARGS);
363 RET parse_float_value(PARAMS) {
365 obj->val = parse_float(ARGS);
369 RET parse_boolean_value(PARAMS) {
371 obj->val = parse_keyword(ARGS, "true", "invalid boolean value"); OR
372 obj->val = parse_keyword(ARGS, "false", "invalid boolean value");
376 RET parse_string_value(PARAMS) {
378 obj->val = parse_string(ARGS);
382 RET parse_null_value(PARAMS) {
384 obj->val = parse_keyword(ARGS, "null", "null expected");
388 RET parse_enum_value(PARAMS) {
390 obj->val = parse_name(ARGS); REQ
391 struct graphql_token *tok = list_top(used, struct graphql_token, list);
392 if (streq(tok->token_string, "true")
393 || streq(tok->token_string, "false")
394 || streq(tok->token_string, "null")) {
395 *err = "enum value cannot be true, false, or null";
401 RET parse_list_value(PARAMS) {
403 parse_punct(ARGS, '['); REQ
404 parse_punct(ARGS, ']');
407 parse_value(ARGS); MSG("expected: value or ']'"); REQ
408 parse_punct(ARGS, ']');
413 RET parse_object_value(PARAMS) {
415 parse_punct(ARGS, '{'); REQ
416 parse_punct(ARGS, '}');
417 struct graphql_object_field *p = NULL;
421 obj->first = p = parse_object_field(ARGS); MSG("expected: object field or '}'"); REQ
423 p->next = parse_object_field(ARGS); MSG("expected: object field or '}'"); REQ
426 parse_punct(ARGS, '}');
431 RET parse_object_field(PARAMS) {
433 obj->name = parse_name(ARGS); REQ
434 parse_punct(ARGS, ':'); REQ
435 obj->val = parse_value(ARGS); REQ
439 RET parse_variable(PARAMS) {
441 parse_punct(ARGS, '$'); REQ
442 obj->name = parse_name(ARGS); REQ
446 RET parse_variable_definitions(PARAMS) {
447 INIT(variable_definitions);
448 parse_punct(ARGS, '('); REQ
449 obj->first = parse_variable_definition(ARGS); REQ
450 struct graphql_variable_definition *p = obj->first;
451 parse_punct(ARGS, ')');
454 p->next = parse_variable_definition(ARGS); MSG("expected: variable definition or ')'"); REQ
456 parse_punct(ARGS, ')');
461 RET parse_variable_definition(PARAMS) {
462 INIT(variable_definition);
463 obj->var = parse_variable(ARGS); REQ
464 parse_punct(ARGS, ':'); REQ
465 obj->type = parse_type(ARGS); REQ
466 obj->default_val = parse_default_value(ARGS); OPT
467 obj->directives = parse_directives(ARGS); OPT
471 RET parse_default_value(PARAMS) {
473 parse_punct(ARGS, '='); REQ
474 obj->val = parse_value(ARGS); REQ
478 RET parse_type(PARAMS) {
480 obj->named = parse_named_type(ARGS);
483 obj->list = parse_list_type(ARGS); OR
484 obj->non_null = parse_non_null_type(ARGS);
489 RET parse_named_type(PARAMS) {
491 obj->name = parse_name(ARGS);
496 RET parse_list_type(PARAMS) {
498 parse_punct(ARGS, '['); REQ
499 parse_type(ARGS); REQ
500 parse_punct(ARGS, ']'); REQ
504 RET parse_non_null_type(PARAMS) {
506 parse_non_null_type_1(ARGS); OR
507 parse_non_null_type_2(ARGS);
511 RET parse_non_null_type_1(PARAMS) {
513 parse_named_type(ARGS); REQ;
514 parse_punct(ARGS, '!'); REQ;
518 RET parse_non_null_type_2(PARAMS) {
520 parse_list_type(ARGS); REQ;
521 parse_punct(ARGS, '!'); REQ;
526 RET parse_directives(PARAMS) {
528 obj->first = parse_directive(ARGS); REQ
529 struct graphql_directive *p = obj->first;
531 p->next = parse_directive(ARGS);
537 RET parse_directive(PARAMS) {
539 parse_punct(ARGS, '@'); REQ
540 obj->name = parse_name(ARGS); REQ
541 obj->args = parse_arguments(ARGS); OPT
546 /* The following functions construct the "leaves" of the abstract syntax tree. */
548 RET parse_keyword(PARAMS, const char *keyword, const char *errmsg) {
549 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
550 if (!tok || tok->token_type != 'a') {
551 *err = errmsg; return NULL;
553 if (!streq(tok->token_string, keyword)) {
554 *err = errmsg; return NULL;
560 // Note: a static buffer is used here.
561 RET parse_punct(PARAMS, int punct) {
562 static char punctbuf[16];
563 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
564 if (!tok || tok->token_type != punct) {
566 sprintf(punctbuf, "expected: '...'");
568 sprintf(punctbuf, "expected: '%c'", punct);
569 *err = punctbuf; return NULL;
575 RET parse_name(PARAMS) {
576 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
577 if (!tok || tok->token_type != 'a') {
578 *err = "name expected"; return NULL;
584 RET parse_int(PARAMS) {
585 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
586 if (!tok || tok->token_type != 'i') {
587 *err = "integer expected"; return NULL;
593 RET parse_float(PARAMS) {
594 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
595 if (!tok || tok->token_type != 'f') {
596 *err = "float expected"; return NULL;
602 RET parse_string(PARAMS) {
603 struct graphql_token *tok = list_top(tokens, struct graphql_token, list);
604 if (!tok || tok->token_type != 's') {
605 *err = "string expected"; return NULL;
612 // Convert input string into tokens.
613 const char *graphql_lex(const char *input, const tal_t *ctx, struct list_head **tokens) {
616 const char *p, *line_beginning;
617 unsigned int line_num = 1;
618 struct list_head *tok_list;
619 struct graphql_token *tok;
621 // Initialize token output list.
622 tok_list = tal(ctx, struct list_head);
625 list_head_init(tok_list);
627 // Note: label and goto are used here like a continue statement except that
628 // it skips iteration, for when characters are fetched in the loop body.
634 // Consume line terminators and increment line counter.
635 if (LINE_TERMINATOR(c)) {
638 if (c0 == 10 || c0 == 13)
640 if (c0 == 13 && c == 10)
642 line_beginning = p - 1;
646 // Consume other ignored tokens.
647 if (COMMA(c) || WHITE_SPACE(c)) {
652 while ((c = *p++) != EOF && !EOF_CHAR(c) && COMMENT_CHAR(c))
657 // Return success when end is reached.
659 return GRAPHQL_SUCCESS;
661 // Punctuator tokens.
664 // Note beginning of token in input.
665 const char *start = p - 1;
667 // Handle the ... multi-character case.
671 return "unrecognized punctuator";
674 return "unrecognized punctuator";
678 tok = tal(tok_list, struct graphql_token);
679 list_add_tail(tok_list, &tok->list);
681 tok->token_specific = c;
682 tok->token_string = NULL;
683 tok->source_line = line_num;
684 tok->source_column = start - line_beginning + 1;
685 tok->source_len = p - start;
687 } else if (NAME_START(c)) {
689 // Name/identifier tokens.
690 tok = tal(tok_list, struct graphql_token);
691 list_add_tail(tok_list, &tok->list);
692 tok->token_type = 'a';
693 tok->token_specific = 'a';
694 // tok->token_string updated below.
695 tok->source_line = line_num;
696 tok->source_column = p - line_beginning;
697 // tok->source_len updated below.
699 // Note the beginning of the name.
700 const char *name_begin = p - 1;
701 const char *name_end;
704 // Consume the rest of the token.
707 } while (NAME_CONTINUE(c));
709 // Note the end of the name and calculate the length.
711 name_len = name_end - name_begin;
712 tok->source_len = name_len;
714 // Copy the token string.
715 tok->token_string = tal_strndup(tok, name_begin, name_len);
719 } else if (DIGIT(c) || c == '-') {
722 const char *num_start = p - 1;
728 return "negative sign must precede a number";
734 return "leading zeros are not allowed";
744 return "invalid float value fractional part";
750 if (c == 'e' || c == 'E') {
753 if (c == '+' || c == '-')
756 return "invalid float value exponent part";
762 if (c == '.' || NAME_START(c))
763 return "invalid numeric value";
765 const char *num_end = p - 1;
766 int num_len = num_end - num_start;
768 tok = tal(tok_list, struct graphql_token);
769 list_add_tail(tok_list, &tok->list);
770 tok->token_type = type;
771 tok->token_string = tal_strndup(tok, num_start, num_len);
772 tok->source_line = line_num;
773 tok->source_column = num_start - line_beginning + 1;
774 tok->source_len = num_len;
778 } else if (c == '"') {
782 const char *str_begin = p - 1;
784 bool str_block = false;
794 if (c == '\"') quotes++; else quotes = 0;
795 if (quotes == 3 && *(p-4) == '\\') quotes = 0;
796 } while (BLOCK_STRING_CHAR(c) && quotes < 3);
803 return "unterminated string or invalid character";
806 return "invalid string termination";
809 return "invalid string termination";
822 if (strchr("\"\\/bfnrtu", c)) {
826 return "invalid unicode escape sequence";
829 return "invalid unicode escape sequence";
832 return "invalid unicode escape sequence";
835 return "invalid unicode escape sequence";
837 c = 'a'; // anything besides a quote to let the loop continue
840 return "invalid string escape sequence";
843 } while (STRING_CHAR(c));
845 return "unterminated string or invalid character";
848 int str_len = str_end - str_begin;
850 tok = tal(tok_list, struct graphql_token);
851 list_add_tail(tok_list, &tok->list);
852 tok->token_type = 's';
853 tok->token_specific = 's';
854 tok->token_string = tal_strndup(tok, str_begin, str_len);
855 tok->source_line = line_num;
856 tok->source_column = str_begin - line_beginning + 1;
857 tok->source_len = str_len;
859 // Process escape sequences. These always shorten the string (so the memory allocation is always enough).
861 char *q = tok->token_string;
866 if (d == '\"') quotes++; else quotes = 0;
867 if (quotes == 3 && *(q-4) == '\\') {
869 rewrite_dest = q - 4;
870 cpystr(rewrite_dest, q - 3);
874 rewrite_dest = q - 1;
878 *rewrite_dest++ = '\"';
879 cpystr(rewrite_dest, q--);
882 *rewrite_dest++ = '\b';
883 cpystr(rewrite_dest, q--);
886 *rewrite_dest++ = '\f';
887 cpystr(rewrite_dest, q--);
890 *rewrite_dest++ = '\n';
891 cpystr(rewrite_dest, q--);
894 *rewrite_dest++ = '\r';
895 cpystr(rewrite_dest, q--);
898 *rewrite_dest++ = '\t';
899 cpystr(rewrite_dest, q--);
902 // Insert escaped character using UTF-8 multi-byte encoding.
903 char buf[] = {*q++, *q++, *q++, *q++, 0};
904 int code_point = strtol(buf, 0, 16);
905 int bytes = utf8_encode(code_point, rewrite_dest);
906 rewrite_dest += bytes;
907 cpystr(rewrite_dest, q--);
911 cpystr(rewrite_dest, --q);
917 // Strip leading lines.
918 q = tok->token_string;
921 while (WHITE_SPACE(d))
923 if (LINE_TERMINATOR(d)) {
924 while (LINE_TERMINATOR(d))
926 cpystr(tok->token_string, q - 1);
927 q = tok->token_string;
932 // Strip trailing lines.
933 q = tok->token_string + strlen(tok->token_string);
936 while (WHITE_SPACE(d))
938 if (LINE_TERMINATOR(d)) {
939 while (LINE_TERMINATOR(d))
946 // Look for common indentation.
947 char *this_indent_start;
948 const char *this_indent_end;
949 const char *common_indent_start = NULL;
950 const char *common_indent_end;
952 q = tok->token_string;
955 this_indent_start = q - 1;
956 while (WHITE_SPACE(d))
958 this_indent_end = q - 1;
959 if (LINE_TERMINATOR(d)) {
960 while (LINE_TERMINATOR(d))
967 if (common_indent_start == NULL) {
968 common_indent_start = this_indent_start;
969 common_indent_end = this_indent_end;
971 for (r = this_indent_start; r < this_indent_end && (r - this_indent_start + common_indent_start < common_indent_end); r++) {
972 if (*r != *(r - this_indent_start + common_indent_start))
975 common_indent_end = r - this_indent_start + common_indent_start;
977 while (!LINE_TERMINATOR(d) && !EOF_CHAR(d))
979 while (LINE_TERMINATOR(d))
985 // Remove common indentation.
986 int common_indent_len = common_indent_end - common_indent_start;
987 if (common_indent_len > 0) {
988 q = tok->token_string;
991 this_indent_start = q - 1;
992 while (WHITE_SPACE(d))
994 this_indent_end = q - 1;
995 if (LINE_TERMINATOR(d)) {
996 while (LINE_TERMINATOR(d))
1003 while (!LINE_TERMINATOR(d) && !EOF_CHAR(d))
1007 cpystr(this_indent_start, this_indent_start + common_indent_len);
1008 q -= common_indent_len;
1011 while (LINE_TERMINATOR(d))
1022 return "invalid source character encountered";
1025 } while (!EOF_CHAR(c));
1027 return "unexpected end-of-input encountered";