Line data Source code
1 : /* valascanner.vala
2 : *
3 : * Copyright (C) 2008-2012 Jürg Billeter
4 : *
5 : * This library is free software; you can redistribute it and/or
6 : * modify it under the terms of the GNU Lesser General Public
7 : * License as published by the Free Software Foundation; either
8 : * version 2.1 of the License, or (at your option) any later version.
9 :
10 : * This library is distributed in the hope that it will be useful,
11 : * but WITHOUT ANY WARRANTY; without even the implied warranty of
12 : * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 : * Lesser General Public License for more details.
14 :
15 : * You should have received a copy of the GNU Lesser General Public
16 : * License along with this library; if not, write to the Free Software
17 : * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
18 : *
19 : * Author:
20 : * Jürg Billeter <j@bitron.ch>
21 : * Jukka-Pekka Iivonen <jp0409@jippii.fi>
22 : */
23 :
24 : using GLib;
25 :
26 : /**
27 : * Lexical scanner for Vala source files.
28 : */
29 14750 : public class Vala.Scanner {
30 102775645 : public SourceFile source_file { get; private set; }
31 :
32 : TokenType previous;
33 : char* current;
34 : char* end;
35 :
36 : int line;
37 : int column;
38 :
39 6606 : Comment _comment;
40 :
41 6606 : Conditional[] conditional_stack;
42 :
43 : struct Conditional {
44 : public bool matched;
45 : public bool else_found;
46 : public bool skip_section;
47 : }
48 :
49 6606 : State[] state_stack;
50 :
51 : enum State {
52 : PARENS,
53 : BRACE,
54 : BRACKET,
55 : TEMPLATE,
56 : TEMPLATE_PART,
57 : REGEX_LITERAL,
58 : VERBATIM_TEMPLATE
59 : }
60 :
61 13212 : public Scanner (SourceFile source_file) {
62 6606 : this.source_file = source_file;
63 :
64 6606 : char* begin = source_file.get_mapped_contents ();
65 6606 : end = begin + source_file.get_mapped_length ();
66 :
67 6606 : current = begin;
68 :
69 6606 : line = 1;
70 6606 : column = 1;
71 : }
72 :
73 8 : public void seek (SourceLocation location) {
74 8 : current = location.pos;
75 8 : line = location.line;
76 8 : column = location.column;
77 :
78 8 : conditional_stack = null;
79 8 : state_stack = null;
80 : }
81 :
82 193748544 : inline bool in_template () {
83 193748544 : return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE);
84 : }
85 :
86 193748557 : inline bool in_verbatim_template () {
87 193748557 : return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.VERBATIM_TEMPLATE);
88 : }
89 :
90 179617845 : inline bool in_template_part () {
91 179617845 : return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE_PART);
92 : }
93 :
94 179617827 : inline bool in_regex_literal () {
95 179617827 : return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.REGEX_LITERAL);
96 : }
97 :
98 684065374 : inline bool is_ident_char (char c) {
99 684065374 : return (c.isalnum () || c == '_');
100 : }
101 :
102 9916 : SourceReference get_source_reference (int offset, int length = 0) {
103 9916 : return new SourceReference (source_file, SourceLocation (current, line, column + offset), SourceLocation (current + length, line, column + offset + length));
104 : }
105 :
106 60 : public TokenType read_regex_token (out SourceLocation token_begin, out SourceLocation token_end) {
107 : TokenType type;
108 60 : char* begin = current;
109 60 : token_begin = SourceLocation (begin, line, column);
110 :
111 60 : int token_length_in_chars = -1;
112 :
113 60 : if (current >= end) {
114 : type = TokenType.EOF;
115 : } else {
116 60 : switch (current[0]) {
117 : case '/':
118 30 : type = TokenType.CLOSE_REGEX_LITERAL;
119 30 : current++;
120 30 : state_stack.length--;
121 30 : var fl_i = false;
122 30 : var fl_s = false;
123 30 : var fl_m = false;
124 30 : var fl_x = false;
125 30 : var fl_o = false;
126 52 : while (current[0] == 'i' || current[0] == 's' || current[0] == 'm' || current[0] == 'x' || current[0] == 'o') {
127 22 : switch (current[0]) {
128 : case 'i':
129 7 : if (fl_i) {
130 1 : Report.error (get_source_reference (token_length_in_chars), "modifier 'i' used more than once");
131 : }
132 : fl_i = true;
133 : break;
134 : case 's':
135 8 : if (fl_s) {
136 1 : Report.error (get_source_reference (token_length_in_chars), "modifier 's' used more than once");
137 : }
138 : fl_s = true;
139 : break;
140 : case 'm':
141 2 : if (fl_m) {
142 1 : Report.error (get_source_reference (token_length_in_chars), "modifier 'm' used more than once");
143 : }
144 : fl_m = true;
145 : break;
146 : case 'x':
147 2 : if (fl_x) {
148 1 : Report.error (get_source_reference (token_length_in_chars), "modifier 'x' used more than once");
149 : }
150 : fl_x = true;
151 : break;
152 : case 'o':
153 3 : if (fl_o) {
154 1 : Report.error (get_source_reference (token_length_in_chars), "modifier 'o' used more than once");
155 : }
156 : fl_o = true;
157 : break;
158 : }
159 22 : current++;
160 22 : token_length_in_chars++;
161 : }
162 : break;
163 : default:
164 512 : type = TokenType.REGEX_LITERAL;
165 : token_length_in_chars = 0;
166 512 : while (current < end && current[0] != '/') {
167 482 : if (current[0] == '\\') {
168 68 : current++;
169 68 : token_length_in_chars++;
170 68 : if (current >= end) {
171 : break;
172 : }
173 :
174 68 : switch (current[0]) {
175 : case '\'':
176 : case '"':
177 : case '\\':
178 : case '/':
179 : case '^':
180 : case '$':
181 : case '.':
182 : case '[':
183 : case ']':
184 : case '{':
185 : case '}':
186 : case '(':
187 : case ')':
188 : case '?':
189 : case '*':
190 : case '+':
191 : case '-':
192 : case '#':
193 : case '&':
194 : case '~':
195 : case ':':
196 : case ';':
197 : case '<':
198 : case '>':
199 : case '|':
200 : case '%':
201 : case '=':
202 : case '@':
203 : case '0':
204 : case 'b':
205 : case 'B':
206 : case 'f':
207 : case 'n':
208 : case 'N':
209 : case 'r':
210 : case 'R':
211 : case 't':
212 : case 'v':
213 : case 'a':
214 : case 'A':
215 : case 'p':
216 : case 'P':
217 : case 'e':
218 : case 'd':
219 : case 'D':
220 : case 's':
221 : case 'S':
222 : case 'w':
223 : case 'W':
224 : case 'G':
225 : case 'z':
226 : case 'Z':
227 58 : current++;
228 58 : token_length_in_chars++;
229 58 : break;
230 : case 'u':
231 : // u escape character has four hex digits
232 2 : current++;
233 2 : token_length_in_chars++;
234 : int digit_length;
235 5 : for (digit_length = 0; current < end && current[0].isxdigit (); digit_length++) {
236 3 : current++;
237 3 : token_length_in_chars++;
238 : }
239 2 : if (digit_length < 1) {
240 1 : Report.error (get_source_reference (token_length_in_chars), "\\u used with no following hex digits");
241 1 : } else if (digit_length < 4) {
242 1 : Report.error (get_source_reference (token_length_in_chars), "incomplete universal character name");
243 : }
244 : break;
245 : case 'x':
246 : // hexadecimal escape character requires two hex digits
247 5 : current++;
248 5 : token_length_in_chars++;
249 : int digit_length;
250 5 : bool empty = true;
251 16 : for (digit_length = 0; current < end && current[0].isxdigit ();) {
252 11 : if (current[0] != '0') {
253 6 : digit_length++;
254 : } else {
255 : empty = false;
256 : }
257 11 : current++;
258 11 : token_length_in_chars++;
259 : }
260 5 : if (empty && digit_length < 1) {
261 1 : Report.error (get_source_reference (token_length_in_chars), "\\x used with no following hex digits");
262 4 : } else if (digit_length > 2) {
263 1 : Report.error (get_source_reference (token_length_in_chars), "hex escape sequence out of range");
264 : }
265 : break;
266 : default:
267 : // back references \1 through \99
268 3 : if (current[0].isdigit ()) {
269 2 : current++;
270 2 : token_length_in_chars++;
271 2 : if (current[0].isdigit ()) {
272 1 : current++;
273 1 : token_length_in_chars++;
274 : }
275 : } else {
276 1 : Report.error (get_source_reference (token_length_in_chars), "invalid escape sequence");
277 : }
278 : break;
279 : }
280 414 : } else if (current[0] == '\n') {
281 : break;
282 : } else {
283 414 : unichar u = ((string) current).get_char_validated ((long) (end - current));
284 414 : if (u != (unichar) (-1)) {
285 414 : current += u.to_utf8 (null);
286 414 : token_length_in_chars++;
287 : } else {
288 0 : current++;
289 0 : Report.error (get_source_reference (token_length_in_chars), "invalid UTF-8 character");
290 : }
291 : }
292 : }
293 30 : if (current >= end || current[0] == '\n') {
294 0 : Report.error (get_source_reference (token_length_in_chars), "syntax error, expected \"");
295 0 : state_stack.length--;
296 0 : return read_token (out token_begin, out token_end);
297 : }
298 : break;
299 : }
300 : }
301 :
302 60 : if (token_length_in_chars < 0) {
303 13 : column += (int) (current - begin);
304 : } else {
305 47 : column += token_length_in_chars;
306 : }
307 :
308 60 : token_end = SourceLocation (current, line, column - 1);
309 :
310 60 : return type;
311 : }
312 :
313 : public static TokenType get_identifier_or_keyword (char* begin, int len) {
314 83183156 : switch (len) {
315 : case 2:
316 298664 : switch (begin[0]) {
317 : case 'a':
318 698 : if (matches (begin, "as")) return TokenType.AS;
319 : break;
320 : case 'd':
321 4027 : if (matches (begin, "do")) return TokenType.DO;
322 : break;
323 : case 'i':
324 85434 : switch (begin[1]) {
325 : case 'f':
326 83183156 : return TokenType.IF;
327 : case 'n':
328 772 : return TokenType.IN;
329 : case 's':
330 1370 : return TokenType.IS;
331 : }
332 : break;
333 : }
334 : break;
335 : case 3:
336 3996653 : switch (begin[0]) {
337 : case 'f':
338 18222 : if (matches (begin, "for")) return TokenType.FOR;
339 : break;
340 : case 'g':
341 438879 : if (matches (begin, "get")) return TokenType.GET;
342 : break;
343 : case 'n':
344 35056 : if (matches (begin, "new")) return TokenType.NEW;
345 : break;
346 : case 'o':
347 535774 : if (matches (begin, "out")) return TokenType.OUT;
348 : break;
349 : case 'r':
350 126182 : if (matches (begin, "ref")) return TokenType.REF;
351 : break;
352 : case 's':
353 380400 : if (matches (begin, "set")) return TokenType.SET;
354 : break;
355 : case 't':
356 21066 : if (matches (begin, "try")) return TokenType.TRY;
357 : break;
358 : case 'v':
359 66465 : if (matches (begin, "var")) return TokenType.VAR;
360 : break;
361 : }
362 : break;
363 : case 4:
364 14453470 : switch (begin[0]) {
365 : case 'b':
366 1575842 : if (matches (begin, "base")) return TokenType.BASE;
367 : break;
368 : case 'c':
369 300588 : if (matches (begin, "case")) return TokenType.CASE;
370 : break;
371 : case 'e':
372 320613 : switch (begin[1]) {
373 : case 'l':
374 52949 : if (matches (begin, "else")) return TokenType.ELSE;
375 : break;
376 : case 'n':
377 221364 : if (matches (begin, "enum")) return TokenType.ENUM;
378 : break;
379 : }
380 : break;
381 : case 'l':
382 175609 : if (matches (begin, "lock")) return TokenType.LOCK;
383 : break;
384 : case 'n':
385 1188753 : if (matches (begin, "null")) return TokenType.NULL;
386 : break;
387 : case 't':
388 895642 : switch (begin[1]) {
389 : case 'h':
390 125375 : if (matches (begin, "this")) return TokenType.THIS;
391 : break;
392 : case 'r':
393 422578 : if (matches (begin, "true")) return TokenType.TRUE;
394 : break;
395 : }
396 : break;
397 : case 'v':
398 2224518 : if (matches (begin, "void")) return TokenType.VOID;
399 : break;
400 : case 'w':
401 70673 : switch (begin[1]) {
402 : case 'e':
403 59444 : if (matches (begin, "weak")) return TokenType.WEAK;
404 : break;
405 : case 'i':
406 150 : if (matches (begin, "with")) return TokenType.WITH;
407 : break;
408 : }
409 : break;
410 : }
411 : break;
412 : case 5:
413 14525277 : switch (begin[0]) {
414 : case 'a':
415 257731 : if (matches (begin, "async")) return TokenType.ASYNC;
416 : break;
417 : case 'b':
418 91252 : if (matches (begin, "break")) return TokenType.BREAK;
419 : break;
420 : case 'c':
421 2829211 : switch (begin[1]) {
422 : case 'a':
423 6347 : if (matches (begin, "catch")) return TokenType.CATCH;
424 : break;
425 : case 'l':
426 443689 : if (matches (begin, "class")) return TokenType.CLASS;
427 : break;
428 : case 'o':
429 667893 : if (matches (begin, "const")) return TokenType.CONST;
430 : break;
431 : }
432 : break;
433 : case 'f':
434 1571095 : if (matches (begin, "false")) return TokenType.FALSE;
435 : break;
436 : case 'o':
437 405801 : if (matches (begin, "owned")) return TokenType.OWNED;
438 : break;
439 : case 't':
440 25857 : if (matches (begin, "throw")) return TokenType.THROW;
441 : break;
442 : case 'u':
443 359808 : if (matches (begin, "using")) return TokenType.USING;
444 : break;
445 : case 'w':
446 21956 : if (matches (begin, "while")) return TokenType.WHILE;
447 : break;
448 : case 'y':
449 4849 : if (matches (begin, "yield")) return TokenType.YIELD;
450 : break;
451 : }
452 : break;
453 : case 6:
454 19237716 : switch (begin[0]) {
455 : case 'd':
456 453281 : if (matches (begin, "delete")) return TokenType.DELETE;
457 : break;
458 : case 'e':
459 117590 : if (matches (begin, "extern")) return TokenType.EXTERN;
460 : break;
461 : case 'i':
462 74011 : if (matches (begin, "inline")) return TokenType.INLINE;
463 : break;
464 : case 'p':
465 9266082 : switch (begin[1]) {
466 : case 'a':
467 16215 : if (matches (begin, "params")) return TokenType.PARAMS;
468 : break;
469 : case 'u':
470 9223572 : if (matches (begin, "public")) return TokenType.PUBLIC;
471 : break;
472 : }
473 : break;
474 : case 'r':
475 333447 : if (matches (begin, "return")) return TokenType.RETURN;
476 : break;
477 : case 's':
478 6061002 : switch (begin[1]) {
479 : case 'e':
480 77403 : if (matches (begin, "sealed")) return TokenType.SEALED;
481 : break;
482 : case 'i':
483 444335 : switch (begin[2]) {
484 : case 'g':
485 144239 : if (matches (begin, "signal")) return TokenType.SIGNAL;
486 : break;
487 : case 'z':
488 296868 : if (matches (begin, "sizeof")) return TokenType.SIZEOF;
489 : break;
490 : }
491 : break;
492 : case 't':
493 5456217 : switch (begin[2]) {
494 : case 'a':
495 1590722 : if (matches (begin, "static")) return TokenType.STATIC;
496 : break;
497 : case 'r':
498 3862127 : if (matches (begin, "struct")) return TokenType.STRUCT;
499 : break;
500 : }
501 : break;
502 : case 'w':
503 115 : if (matches (begin, "switch")) return TokenType.SWITCH;
504 : break;
505 : }
506 : break;
507 : case 't':
508 952806 : switch (begin[1]) {
509 : case 'h':
510 888172 : if (matches (begin, "throws")) return TokenType.THROWS;
511 : break;
512 : case 'y':
513 150 : if (matches (begin, "typeof")) return TokenType.TYPEOF;
514 : break;
515 : }
516 : break;
517 : case 'u':
518 319771 : if (matches (begin, "unlock")) return TokenType.UNLOCK;
519 : break;
520 : }
521 : break;
522 : case 7:
523 6977012 : switch (begin[0]) {
524 : case 'd':
525 30086 : switch (begin[1]) {
526 : case 'e':
527 17587 : if (matches (begin, "default")) return TokenType.DEFAULT;
528 : break;
529 : case 'y':
530 22 : if (matches (begin, "dynamic")) return TokenType.DYNAMIC;
531 : break;
532 : }
533 : break;
534 : case 'e':
535 41623 : if (matches (begin, "ensures")) return TokenType.ENSURES;
536 : break;
537 : case 'f':
538 27229 : switch (begin[1]) {
539 : case 'i':
540 4671 : if (matches (begin, "finally")) return TokenType.FINALLY;
541 : break;
542 : case 'o':
543 9943 : if (matches (begin, "foreach")) return TokenType.FOREACH;
544 : break;
545 : }
546 : break;
547 : case 'p':
548 52610 : switch (begin[1]) {
549 : case 'r':
550 32641 : if (matches (begin, "private")) return TokenType.PRIVATE;
551 : break;
552 : case 'a':
553 10110 : if (matches (begin, "partial")) return TokenType.PARTIAL;
554 : break;
555 : }
556 : break;
557 : case 'u':
558 948178 : if (matches (begin, "unowned")) return TokenType.UNOWNED;
559 : break;
560 : case 'v':
561 520915 : if (matches (begin, "virtual")) return TokenType.VIRTUAL;
562 : break;
563 : }
564 : break;
565 : case 8:
566 2714164 : switch (begin[0]) {
567 : case 'a':
568 446108 : if (matches (begin, "abstract")) return TokenType.ABSTRACT;
569 : break;
570 : case 'c':
571 86401 : if (matches (begin, "continue")) return TokenType.CONTINUE;
572 : break;
573 : case 'd':
574 219752 : if (matches (begin, "delegate")) return TokenType.DELEGATE;
575 : break;
576 : case 'i':
577 142205 : if (matches (begin, "internal")) return TokenType.INTERNAL;
578 : break;
579 : case 'o':
580 42443 : if (matches (begin, "override")) return TokenType.OVERRIDE;
581 : break;
582 : case 'r':
583 19242 : if (matches (begin, "requires")) return TokenType.REQUIRES;
584 : break;
585 : case 'v':
586 44766 : if (matches (begin, "volatile")) return TokenType.VOLATILE;
587 : break;
588 : }
589 : break;
590 : case 9:
591 2153566 : switch (begin[0]) {
592 : case 'c':
593 311328 : if (matches (begin, "construct")) return TokenType.CONSTRUCT;
594 : break;
595 : case 'i':
596 115955 : if (matches (begin, "interface")) return TokenType.INTERFACE;
597 : break;
598 : case 'n':
599 95184 : if (matches (begin, "namespace")) return TokenType.NAMESPACE;
600 : break;
601 : case 'p':
602 114022 : if (matches (begin, "protected")) return TokenType.PROTECTED;
603 : break;
604 : }
605 : break;
606 : case 11:
607 3319252 : if (matches (begin, "errordomain")) return TokenType.ERRORDOMAIN;
608 : break;
609 : }
610 : return TokenType.IDENTIFIER;
611 : }
612 :
613 683103 : TokenType read_number () {
614 683103 : var type = TokenType.INTEGER_LITERAL;
615 :
616 : // integer part
617 683103 : if (current < end - 2 && current[0] == '0') {
618 213252 : switch (current[1]) {
619 : case 'x':
620 : case 'X':
621 : // hexadecimal literal
622 6191 : current += 2;
623 18602 : while (current < end && current[0].isxdigit ()) {
624 12411 : current++;
625 : }
626 : // fractional part
627 : // hexadecimal fractional part
628 6191 : if (current < end - 1 && current[0] == '.' && current[1].isxdigit ()) {
629 12 : type = TokenType.REAL_LITERAL;
630 12 : current++;
631 36 : while (current < end && current[0].isxdigit ()) {
632 24 : current++;
633 : }
634 : }
635 : // hexadecimal exponent part
636 6191 : if (current < end && current[0].tolower () == 'p') {
637 10 : type = TokenType.REAL_LITERAL;
638 10 : current++;
639 10 : if (current < end && (current[0] == '+' || current[0] == '-')) {
640 0 : current++;
641 : }
642 19 : while (current < end && current[0].isdigit ()) {
643 9 : current++;
644 : }
645 6181 : } else if (type == TokenType.REAL_LITERAL) {
646 2 : Report.error (get_source_reference (1), "hexadecimal floating constants require an exponent");
647 : }
648 : break;
649 : case 'b':
650 : case 'B':
651 : case 'o':
652 : case 'O':
653 : // binary integer literal or octal integer literal
654 17 : current += 2;
655 365 : while (current < end && current[0].isdigit ()) {
656 348 : current++;
657 : }
658 : break;
659 : default:
660 : // decimal number (also may be octal integer)
661 418726 : while (current < end && current[0].isdigit ()) {
662 211682 : current++;
663 : }
664 : break;
665 : }
666 : } else {
667 : // decimal number
668 1015117 : while (current < end && current[0].isdigit ()) {
669 545266 : current++;
670 : }
671 : }
672 :
673 : // fractional part
674 683103 : if (current < end - 1 && current[0] == '.' && current[1].isdigit ()) {
675 276580 : type = TokenType.REAL_LITERAL;
676 276580 : current++;
677 649924 : while (current < end && current[0].isdigit ()) {
678 373344 : current++;
679 : }
680 : }
681 :
682 : // exponent part
683 683103 : if (current < end && current[0].tolower () == 'e') {
684 7 : type = TokenType.REAL_LITERAL;
685 7 : current++;
686 7 : if (current < end && (current[0] == '+' || current[0] == '-')) {
687 6 : current++;
688 : }
689 12 : while (current < end && current[0].isdigit ()) {
690 5 : current++;
691 : }
692 : }
693 :
694 : // type suffix
695 683103 : if (current < end) {
696 683103 : bool real_literal = (type == TokenType.REAL_LITERAL);
697 :
698 683103 : switch (current[0]) {
699 : case 'l':
700 : case 'L':
701 26 : if (type == TokenType.INTEGER_LITERAL) {
702 26 : current++;
703 26 : if (current < end && current[0].tolower () == 'l') {
704 24 : current++;
705 : }
706 : }
707 : break;
708 : case 'u':
709 : case 'U':
710 4641 : if (type == TokenType.INTEGER_LITERAL) {
711 4641 : current++;
712 4641 : if (current < end && current[0].tolower () == 'l') {
713 6 : current++;
714 6 : if (current < end && current[0].tolower () == 'l') {
715 3 : current++;
716 : }
717 : }
718 : }
719 : break;
720 : case 'f':
721 : case 'F':
722 : case 'd':
723 : case 'D':
724 128 : type = TokenType.REAL_LITERAL;
725 128 : current++;
726 128 : break;
727 : }
728 :
729 683103 : if (!real_literal && is_ident_char (current[0])) {
730 : // allow identifiers to start with a digit
731 : // as long as they contain at least one char
732 11 : while (current < end && is_ident_char (current[0])) {
733 8 : current++;
734 : }
735 : type = TokenType.IDENTIFIER;
736 : }
737 : }
738 :
739 : return type;
740 : }
741 :
742 28 : public TokenType read_template_token (out SourceLocation token_begin, out SourceLocation token_end) {
743 28 : bool is_verbatim = in_verbatim_template ();
744 : TokenType type;
745 28 : char* begin = current;
746 28 : token_begin = SourceLocation (begin, line, column);
747 :
748 28 : int token_length_in_chars = -1;
749 :
750 28 : if (current >= end) {
751 : type = TokenType.EOF;
752 : } else {
753 28 : switch (current[0]) {
754 : case '"':
755 8 : if (is_verbatim) {
756 4 : if (current < end -2 && current[1] == '"' && current[2] == '"' && current[3] != '"') {
757 4 : type = TokenType.CLOSE_TEMPLATE;
758 4 : current += 3;
759 4 : state_stack.length--;
760 : } else {
761 0 : type = TokenType.VERBATIM_TEMPLATE_STRING_LITERAL;
762 0 : current++;
763 0 : token_length_in_chars++;
764 0 : state_stack += State.TEMPLATE_PART;
765 : }
766 : } else {
767 4 : type = TokenType.CLOSE_TEMPLATE;
768 4 : current++;
769 4 : state_stack.length--;
770 : }
771 : break;
772 : case '$':
773 10 : token_begin.pos++; // $ is not part of following token
774 10 : current++;
775 10 : if (current[0].isalpha () || current[0] == '_') {
776 : int len = 0;
777 20 : while (current < end && is_ident_char (current[0])) {
778 14 : current++;
779 14 : len++;
780 : }
781 6 : type = TokenType.IDENTIFIER;
782 6 : state_stack += State.TEMPLATE_PART;
783 4 : } else if (current[0] == '(') {
784 2 : current++;
785 2 : column += 2;
786 4 : state_stack += State.PARENS;
787 2 : return read_token (out token_begin, out token_end);
788 2 : } else if (current[0] == '$') {
789 2 : type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
790 2 : current++;
791 2 : state_stack += State.TEMPLATE_PART;
792 : } else {
793 0 : Report.error (get_source_reference (1), "unexpected character");
794 0 : return read_template_token (out token_begin, out token_end);
795 : }
796 : break;
797 : default:
798 10 : type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
799 10 : token_length_in_chars = 0;
800 29 : while (current < end && current[0] != '"' && current[0] != '$') {
801 19 : if (current[0] == '\\' && !is_verbatim) {
802 0 : current++;
803 0 : token_length_in_chars++;
804 0 : if (current >= end) {
805 : break;
806 : }
807 :
808 0 : switch (current[0]) {
809 : case '\'':
810 : case '"':
811 : case '\\':
812 : case '0':
813 : case 'b':
814 : case 'f':
815 : case 'n':
816 : case 'r':
817 : case 't':
818 : case 'v':
819 0 : current++;
820 0 : token_length_in_chars++;
821 0 : break;
822 : case 'u':
823 : // u escape character has four hex digits
824 0 : current++;
825 0 : token_length_in_chars++;
826 : int digit_length;
827 0 : for (digit_length = 0; current < end && current[0].isxdigit (); digit_length++) {
828 0 : current++;
829 0 : token_length_in_chars++;
830 : }
831 0 : if (digit_length < 1) {
832 0 : Report.error (get_source_reference (token_length_in_chars), "\\u used with no following hex digits");
833 0 : } else if (digit_length < 4) {
834 0 : Report.error (get_source_reference (token_length_in_chars), "incomplete universal character name");
835 : }
836 : break;
837 : case 'x':
838 : // hexadecimal escape character requires two hex digits
839 0 : current++;
840 0 : token_length_in_chars++;
841 : int digit_length;
842 0 : bool empty = true;
843 0 : for (digit_length = 0; current < end && current[0].isxdigit ();) {
844 0 : if (current[0] != '0') {
845 0 : digit_length++;
846 : } else {
847 : empty = false;
848 : }
849 0 : current++;
850 0 : token_length_in_chars++;
851 : }
852 0 : if (empty && digit_length < 1) {
853 0 : Report.error (get_source_reference (token_length_in_chars), "\\x used with no following hex digits");
854 0 : } else if (digit_length > 2) {
855 0 : Report.error (get_source_reference (token_length_in_chars), "hex escape sequence out of range");
856 : }
857 : break;
858 : default:
859 0 : Report.error (get_source_reference (token_length_in_chars), "invalid escape sequence");
860 : break;
861 : }
862 19 : } else if (current[0] == '\n') {
863 3 : current++;
864 3 : line++;
865 3 : column = 1;
866 3 : token_length_in_chars = 1;
867 : } else {
868 16 : unichar u = ((string) current).get_char_validated ((long) (end - current));
869 16 : if (u != (unichar) (-1)) {
870 16 : current += u.to_utf8 (null);
871 16 : token_length_in_chars++;
872 : } else {
873 0 : current++;
874 0 : Report.error (get_source_reference (token_length_in_chars), "invalid UTF-8 character");
875 : }
876 : }
877 : }
878 10 : if (current >= end) {
879 0 : Report.error (get_source_reference (token_length_in_chars), "syntax error, expected \"");
880 0 : state_stack.length--;
881 0 : return read_token (out token_begin, out token_end);
882 : }
883 20 : state_stack += State.TEMPLATE_PART;
884 10 : break;
885 : }
886 : }
887 :
888 26 : if (token_length_in_chars < 0) {
889 16 : column += (int) (current - begin);
890 : } else {
891 10 : column += token_length_in_chars;
892 : }
893 :
894 26 : token_end = SourceLocation (current, line, column - 1);
895 :
896 26 : return type;
897 : }
898 :
899 179617873 : public TokenType read_token (out SourceLocation token_begin, out SourceLocation token_end) {
900 179617873 : if (in_template () || in_verbatim_template ()) {
901 28 : return read_template_token (out token_begin, out token_end);
902 179617845 : } else if (in_template_part ()) {
903 18 : state_stack.length--;
904 :
905 18 : token_begin = SourceLocation (current, line, column);
906 18 : token_end = SourceLocation (current, line, column - 1);
907 :
908 18 : return TokenType.COMMA;
909 179617827 : } else if (in_regex_literal ()) {
910 60 : return read_regex_token (out token_begin, out token_end);
911 : }
912 :
913 179617767 : space ();
914 :
915 : TokenType type;
916 179617767 : char* begin = current;
917 179617767 : token_begin = SourceLocation (begin, line, column);
918 :
919 179617767 : int token_length_in_chars = -1;
920 :
921 179617767 : if (current >= end) {
922 : type = TokenType.EOF;
923 179611160 : } else if (current[0].isalpha () || current[0] == '_') {
924 : int len = 0;
925 682472661 : while (current < end && is_ident_char (current[0])) {
926 599323761 : current++;
927 599323761 : len++;
928 : }
929 83148900 : type = get_identifier_or_keyword (begin, len);
930 96462260 : } else if (current[0] == '@') {
931 163298 : if (current < end - 1 && current[1] == '"') {
932 8 : current += 1;
933 8 : if (current < end - 5 && current[1] == '"' && current[2] == '"') {
934 4 : current += 3;
935 8 : state_stack += State.VERBATIM_TEMPLATE;
936 : } else {
937 4 : current += 1;
938 8 : state_stack += State.TEMPLATE;
939 : }
940 : type = TokenType.OPEN_TEMPLATE;
941 : } else {
942 163290 : token_begin.pos++; // @ is not part of the identifier
943 163290 : current++;
944 163290 : int len = 0;
945 842558 : while (current < end && is_ident_char (current[0])) {
946 679268 : current++;
947 679268 : len++;
948 : }
949 : type = TokenType.IDENTIFIER;
950 : }
951 96298962 : } else if (current[0].isdigit ()) {
952 683103 : type = read_number ();
953 : } else {
954 95615859 : switch (current[0]) {
955 : case '{':
956 1623611 : type = TokenType.OPEN_BRACE;
957 1623611 : current++;
958 1623611 : state_stack += State.BRACE;
959 1623611 : break;
960 : case '}':
961 1623610 : type = TokenType.CLOSE_BRACE;
962 1623610 : current++;
963 1623610 : if (state_stack.length > 0) {
964 1623603 : state_stack.length--;
965 : }
966 : break;
967 : case '(':
968 14130669 : type = TokenType.OPEN_PARENS;
969 14130669 : current++;
970 14130669 : state_stack += State.PARENS;
971 14130669 : break;
972 : case ')':
973 14130671 : type = TokenType.CLOSE_PARENS;
974 14130671 : current++;
975 14130671 : if (state_stack.length > 0) {
976 14130671 : state_stack.length--;
977 : }
978 14130671 : if (in_template () || in_verbatim_template ()) {
979 : type = TokenType.COMMA;
980 : }
981 : break;
982 : case '[':
983 7942299 : type = TokenType.OPEN_BRACKET;
984 7942299 : current++;
985 7942299 : state_stack += State.BRACKET;
986 7942299 : break;
987 : case ']':
988 7942299 : type = TokenType.CLOSE_BRACKET;
989 7942299 : current++;
990 7942299 : if (state_stack.length > 0) {
991 7942299 : state_stack.length--;
992 : }
993 : break;
994 : case '.':
995 5711673 : type = TokenType.DOT;
996 5711673 : current++;
997 5711673 : if (current < end - 1) {
998 5711673 : if (current[0] == '.' && current[1] == '.') {
999 126668 : type = TokenType.ELLIPSIS;
1000 126668 : current += 2;
1001 : }
1002 : }
1003 : break;
1004 : case ':':
1005 304799 : type = TokenType.COLON;
1006 304799 : current++;
1007 304799 : if (current < end && current[0] == ':') {
1008 18 : type = TokenType.DOUBLE_COLON;
1009 18 : current++;
1010 : }
1011 : break;
1012 : case ',':
1013 9526910 : type = TokenType.COMMA;
1014 9526910 : current++;
1015 9526910 : break;
1016 : case ';':
1017 9245971 : type = TokenType.SEMICOLON;
1018 9245971 : current++;
1019 9245971 : break;
1020 : case '#':
1021 2 : type = TokenType.HASH;
1022 2 : current++;
1023 2 : break;
1024 : case '?':
1025 1919944 : type = TokenType.INTERR;
1026 1919944 : current++;
1027 1919944 : if (current < end && current[0] == '?') {
1028 62 : type = TokenType.OP_COALESCING;
1029 62 : current++;
1030 : }
1031 : break;
1032 : case '|':
1033 11622 : type = TokenType.BITWISE_OR;
1034 11622 : current++;
1035 11622 : if (current < end) {
1036 11622 : switch (current[0]) {
1037 : case '=':
1038 100 : type = TokenType.ASSIGN_BITWISE_OR;
1039 100 : current++;
1040 100 : break;
1041 : case '|':
1042 11445 : type = TokenType.OP_OR;
1043 11445 : current++;
1044 11445 : break;
1045 : }
1046 : }
1047 : break;
1048 : case '&':
1049 33853 : type = TokenType.BITWISE_AND;
1050 33853 : current++;
1051 33853 : if (current < end) {
1052 33853 : switch (current[0]) {
1053 : case '=':
1054 2 : type = TokenType.ASSIGN_BITWISE_AND;
1055 2 : current++;
1056 2 : break;
1057 : case '&':
1058 30727 : type = TokenType.OP_AND;
1059 30727 : current++;
1060 30727 : break;
1061 : }
1062 : }
1063 : break;
1064 : case '^':
1065 7 : type = TokenType.CARRET;
1066 7 : current++;
1067 7 : if (current < end && current[0] == '=') {
1068 2 : type = TokenType.ASSIGN_BITWISE_XOR;
1069 2 : current++;
1070 : }
1071 : break;
1072 : case '~':
1073 27 : type = TokenType.TILDE;
1074 27 : current++;
1075 27 : break;
1076 : case '=':
1077 11008861 : type = TokenType.ASSIGN;
1078 11008861 : current++;
1079 11008861 : if (current < end) {
1080 11008861 : switch (current[0]) {
1081 : case '=':
1082 52315 : type = TokenType.OP_EQ;
1083 52315 : current++;
1084 52315 : break;
1085 : case '>':
1086 6475 : type = TokenType.LAMBDA;
1087 6475 : current++;
1088 6475 : break;
1089 : }
1090 : }
1091 : break;
1092 : case '<':
1093 697421 : type = TokenType.OP_LT;
1094 697421 : current++;
1095 697421 : if (current < end) {
1096 697421 : switch (current[0]) {
1097 : case '=':
1098 18455 : type = TokenType.OP_LE;
1099 18455 : current++;
1100 18455 : break;
1101 : case '<':
1102 24 : type = TokenType.OP_SHIFT_LEFT;
1103 24 : current++;
1104 24 : if (current < end && current[0] == '=') {
1105 2 : type = TokenType.ASSIGN_SHIFT_LEFT;
1106 2 : current++;
1107 : }
1108 : break;
1109 : }
1110 : }
1111 : break;
1112 : case '>':
1113 682134 : type = TokenType.OP_GT;
1114 682134 : current++;
1115 682134 : if (current < end && current[0] == '=') {
1116 16876 : type = TokenType.OP_GE;
1117 16876 : current++;
1118 : }
1119 : break;
1120 : case '!':
1121 114968 : type = TokenType.OP_NEG;
1122 114968 : current++;
1123 114968 : if (current < end && current[0] == '=') {
1124 66338 : type = TokenType.OP_NE;
1125 66338 : current++;
1126 : }
1127 : break;
1128 : case '+':
1129 80933 : type = TokenType.PLUS;
1130 80933 : current++;
1131 80933 : if (current < end) {
1132 80933 : switch (current[0]) {
1133 : case '=':
1134 6285 : type = TokenType.ASSIGN_ADD;
1135 6285 : current++;
1136 6285 : break;
1137 : case '+':
1138 5036 : type = TokenType.OP_INC;
1139 5036 : current++;
1140 5036 : break;
1141 : }
1142 : }
1143 : break;
1144 : case '-':
1145 169379 : type = TokenType.MINUS;
1146 169379 : current++;
1147 169379 : if (current < end) {
1148 169379 : switch (current[0]) {
1149 : case '=':
1150 14 : type = TokenType.ASSIGN_SUB;
1151 14 : current++;
1152 14 : break;
1153 : case '-':
1154 1629 : type = TokenType.OP_DEC;
1155 1629 : current++;
1156 1629 : break;
1157 : case '>':
1158 1538 : type = TokenType.OP_PTR;
1159 1538 : current++;
1160 1538 : break;
1161 : }
1162 : }
1163 : break;
1164 : case '*':
1165 633903 : type = TokenType.STAR;
1166 633903 : current++;
1167 633903 : if (current < end && current[0] == '=') {
1168 6 : type = TokenType.ASSIGN_MUL;
1169 6 : current++;
1170 : }
1171 : break;
1172 : case '/':
1173 47 : switch (previous) {
1174 : case TokenType.ASSIGN:
1175 : case TokenType.COMMA:
1176 : case TokenType.MINUS:
1177 : case TokenType.OP_AND:
1178 : case TokenType.OP_COALESCING:
1179 : case TokenType.OP_EQ:
1180 : case TokenType.OP_GE:
1181 : case TokenType.OP_GT:
1182 : case TokenType.OP_LE:
1183 : case TokenType.OP_LT:
1184 : case TokenType.OP_NE:
1185 : case TokenType.OP_NEG:
1186 : case TokenType.OP_OR:
1187 : case TokenType.OPEN_BRACE:
1188 : case TokenType.OPEN_PARENS:
1189 : case TokenType.PLUS:
1190 : case TokenType.RETURN:
1191 30 : type = TokenType.OPEN_REGEX_LITERAL;
1192 60 : state_stack += State.REGEX_LITERAL;
1193 30 : current++;
1194 30 : break;
1195 : default:
1196 17 : type = TokenType.DIV;
1197 17 : current++;
1198 17 : if (current < end && current[0] == '=') {
1199 5 : type = TokenType.ASSIGN_DIV;
1200 5 : current++;
1201 : }
1202 : break;
1203 : }
1204 : break;
1205 : case '%':
1206 16 : type = TokenType.PERCENT;
1207 16 : current++;
1208 16 : if (current < end && current[0] == '=') {
1209 5 : type = TokenType.ASSIGN_PERCENT;
1210 5 : current++;
1211 : }
1212 : break;
1213 : case '\'':
1214 : case '"':
1215 8080230 : if (begin[0] == '\'') {
1216 : type = TokenType.CHARACTER_LITERAL;
1217 8075054 : } else if (current < end - 6 && begin[1] == '"' && begin[2] == '"') {
1218 28 : type = TokenType.VERBATIM_STRING_LITERAL;
1219 28 : token_length_in_chars = 6;
1220 28 : current += 3;
1221 2789 : while (current < end - 4) {
1222 2789 : if (current[0] == '"' && current[1] == '"' && current[2] == '"' && current[3] != '"') {
1223 : break;
1224 2761 : } else if (current[0] == '\n') {
1225 44 : current++;
1226 44 : line++;
1227 44 : column = 1;
1228 44 : token_length_in_chars = 3;
1229 : } else {
1230 2717 : unichar u = ((string) current).get_char_validated ((long) (end - current));
1231 2717 : if (u != (unichar) (-1)) {
1232 2717 : current += u.to_utf8 (null);
1233 2717 : token_length_in_chars++;
1234 : } else {
1235 0 : Report.error (get_source_reference (token_length_in_chars), "invalid UTF-8 character");
1236 : }
1237 : }
1238 : }
1239 28 : if (current[0] == '"' && current[1] == '"' && current[2] == '"') {
1240 28 : current += 3;
1241 : } else {
1242 0 : Report.error (get_source_reference (token_length_in_chars), "syntax error, expected \"\"\"");
1243 : }
1244 : break;
1245 : } else {
1246 : type = TokenType.STRING_LITERAL;
1247 : }
1248 8080202 : token_length_in_chars = 2;
1249 8080202 : current++;
1250 95603178 : while (current < end && current[0] != begin[0]) {
1251 87522976 : if (current[0] == '\\') {
1252 15096 : current++;
1253 15096 : token_length_in_chars++;
1254 15096 : if (current >= end) {
1255 : break;
1256 : }
1257 :
1258 15096 : switch (current[0]) {
1259 : case '\'':
1260 : case '"':
1261 : case '\\':
1262 : case '0':
1263 : case 'b':
1264 : case 'f':
1265 : case 'n':
1266 : case 'r':
1267 : case 't':
1268 : case 'v':
1269 : case '$':
1270 15077 : current++;
1271 15077 : token_length_in_chars++;
1272 15077 : break;
1273 : case 'u':
1274 : // u escape character has four hex digits
1275 3 : current++;
1276 3 : token_length_in_chars++;
1277 : int digit_length;
1278 10 : for (digit_length = 0; current < end && current[0].isxdigit (); digit_length++) {
1279 7 : current++;
1280 7 : token_length_in_chars++;
1281 : }
1282 3 : if (digit_length < 1) {
1283 1 : Report.error (get_source_reference (token_length_in_chars), "\\u used with no following hex digits");
1284 2 : } else if (digit_length < 4) {
1285 1 : Report.error (get_source_reference (token_length_in_chars), "incomplete universal character name");
1286 : }
1287 : break;
1288 : case 'x':
1289 : // hexadecimal escape character requires two hex digits
1290 16 : current++;
1291 16 : token_length_in_chars++;
1292 : int digit_length;
1293 16 : bool empty = true;
1294 48 : for (digit_length = 0; current < end && current[0].isxdigit ();) {
1295 32 : if (current[0] != '0') {
1296 26 : digit_length++;
1297 : } else {
1298 : empty = false;
1299 : }
1300 32 : current++;
1301 32 : token_length_in_chars++;
1302 : }
1303 16 : if (empty && digit_length < 1) {
1304 1 : Report.error (get_source_reference (token_length_in_chars), "\\x used with no following hex digits");
1305 15 : } else if (digit_length > 2) {
1306 1 : Report.error (get_source_reference (token_length_in_chars), "hex escape sequence out of range");
1307 : }
1308 : break;
1309 : default:
1310 0 : Report.error (get_source_reference (token_length_in_chars), "invalid escape sequence");
1311 : break;
1312 : }
1313 87507880 : } else if (current[0] == '\n') {
1314 0 : current++;
1315 0 : line++;
1316 0 : column = 1;
1317 0 : token_length_in_chars = 1;
1318 : } else {
1319 87507880 : unichar u = ((string) current).get_char_validated ((long) (end - current));
1320 87507880 : if (u != (unichar) (-1)) {
1321 87507880 : current += u.to_utf8 (null);
1322 87507880 : token_length_in_chars++;
1323 : } else {
1324 0 : current++;
1325 0 : Report.error (get_source_reference (token_length_in_chars), "invalid UTF-8 character");
1326 : }
1327 : }
1328 87522976 : if (current < end && begin[0] == '\'' && current[0] != '\'') {
1329 : // multiple characters in single character literal
1330 0 : Report.error (get_source_reference (token_length_in_chars), "invalid character literal");
1331 : }
1332 : }
1333 8080202 : if (current < end) {
1334 8080202 : current++;
1335 : } else {
1336 0 : Report.error (get_source_reference (token_length_in_chars), "syntax error, expected %c", begin[0]);
1337 : }
1338 : break;
1339 : default:
1340 0 : unichar u = ((string) current).get_char_validated ((long) (end - current));
1341 0 : if (u != (unichar) (-1)) {
1342 0 : current += u.to_utf8 (null);
1343 0 : Report.error (get_source_reference (0), "syntax error, unexpected character");
1344 : } else {
1345 0 : current++;
1346 0 : Report.error (get_source_reference (0), "invalid UTF-8 character");
1347 : }
1348 0 : column++;
1349 0 : return read_token (out token_begin, out token_end);
1350 : }
1351 : }
1352 :
1353 179617767 : if (token_length_in_chars < 0) {
1354 171537537 : column += (int) (current - begin);
1355 : } else {
1356 8080230 : column += token_length_in_chars;
1357 : }
1358 :
1359 179617767 : token_end = SourceLocation (current, line, column - 1);
1360 179617767 : previous = type;
1361 :
1362 179617767 : return type;
1363 : }
1364 :
1365 35892440 : static bool matches (char* begin, string keyword) {
1366 35892440 : char* keyword_array = (char*) keyword;
1367 35892440 : long len = keyword.length;
1368 179788670 : for (int i = 0; i < len; i++) {
1369 157367985 : if (begin[i] != keyword_array[i]) {
1370 : return false;
1371 : }
1372 : }
1373 : return true;
1374 : }
1375 :
1376 347163 : bool pp_whitespace () {
1377 : bool found = false;
1378 379310 : while (current < end && current[0].isspace () && current[0] != '\n') {
1379 32147 : found = true;
1380 32147 : current++;
1381 32147 : column++;
1382 : }
1383 : return found;
1384 : }
1385 :
1386 315016 : void pp_space () {
1387 347163 : while (pp_whitespace () || comment ()) {
1388 : }
1389 : }
1390 :
1391 93203 : void pp_directive () {
1392 : // hash sign
1393 93203 : current++;
1394 93203 : column++;
1395 :
1396 93203 : if (line == 1 && column == 2 && current < end && current[0] == '!') {
1397 : // hash bang: #!
1398 : // skip until end of line or end of file
1399 0 : while (current < end && current[0] != '\n') {
1400 0 : current++;
1401 : }
1402 : return;
1403 : }
1404 :
1405 93203 : pp_space ();
1406 :
1407 93203 : char* begin = current;
1408 93203 : int len = 0;
1409 433937 : while (current < end && current[0].isalnum ()) {
1410 340734 : current++;
1411 340734 : column++;
1412 340734 : len++;
1413 : }
1414 :
1415 93203 : if (len == 2 && matches (begin, "if")) {
1416 32078 : parse_pp_if ();
1417 61125 : } else if (len == 4 && matches (begin, "elif")) {
1418 55 : parse_pp_elif ();
1419 61070 : } else if (len == 4 && matches (begin, "else")) {
1420 28990 : parse_pp_else ();
1421 32080 : } else if (len == 5 && matches (begin, "endif")) {
1422 32079 : parse_pp_endif ();
1423 : } else {
1424 1 : Report.error (get_source_reference (-len, len), "syntax error, invalid preprocessing directive");
1425 : }
1426 :
1427 93203 : if (conditional_stack.length > 0
1428 61121 : && conditional_stack[conditional_stack.length - 1].skip_section) {
1429 : // skip lines until next preprocessing directive
1430 : bool bol = false;
1431 6188178 : while (current < end) {
1432 6188178 : if (bol && current[0] == '#') {
1433 : // go back to begin of line
1434 30594 : current -= (column - 1);
1435 30594 : column = 1;
1436 30594 : return;
1437 : }
1438 6157584 : if (current[0] == '\n') {
1439 153628 : line++;
1440 153628 : column = 0;
1441 153628 : bol = true;
1442 6003956 : } else if (!current[0].isspace ()) {
1443 5173897 : bol = false;
1444 : }
1445 6157584 : current++;
1446 6157584 : column++;
1447 : }
1448 : }
1449 : }
1450 :
1451 93202 : void pp_eol () {
1452 93202 : pp_space ();
1453 93202 : if (current >= end || current[0] != '\n') {
1454 0 : Report.error (get_source_reference (0), "syntax error, expected newline");
1455 : }
1456 : }
1457 :
1458 32078 : void parse_pp_if () {
1459 32078 : pp_space ();
1460 :
1461 32078 : bool condition = parse_pp_expression ();
1462 :
1463 32078 : pp_eol ();
1464 :
1465 64156 : conditional_stack += Conditional ();
1466 :
1467 32078 : if (condition && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) {
1468 : // condition true => process code within if
1469 13682 : conditional_stack[conditional_stack.length - 1].matched = true;
1470 : } else {
1471 : // skip lines until next preprocessing directive
1472 18396 : conditional_stack[conditional_stack.length - 1].skip_section = true;
1473 : }
1474 : }
1475 :
1476 55 : void parse_pp_elif () {
1477 55 : pp_space ();
1478 :
1479 55 : bool condition = parse_pp_expression ();
1480 :
1481 55 : pp_eol ();
1482 :
1483 55 : if (conditional_stack.length == 0 || conditional_stack[conditional_stack.length - 1].else_found) {
1484 1 : Report.error (get_source_reference (0), "syntax error, unexpected #elif");
1485 1 : return;
1486 : }
1487 :
1488 54 : if (condition && !conditional_stack[conditional_stack.length - 1].matched
1489 0 : && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) {
1490 : // condition true => process code within if
1491 0 : conditional_stack[conditional_stack.length - 1].matched = true;
1492 0 : conditional_stack[conditional_stack.length - 1].skip_section = false;
1493 : } else {
1494 : // skip lines until next preprocessing directive
1495 54 : conditional_stack[conditional_stack.length - 1].skip_section = true;
1496 : }
1497 : }
1498 :
1499 28991 : void parse_pp_else () {
1500 28990 : pp_eol ();
1501 :
1502 28990 : if (conditional_stack.length == 0 || conditional_stack[conditional_stack.length - 1].else_found) {
1503 1 : Report.error (get_source_reference (0), "syntax error, unexpected #else");
1504 1 : return;
1505 : }
1506 :
1507 28989 : if (!conditional_stack[conditional_stack.length - 1].matched
1508 16845 : && (conditional_stack.length == 1 || !conditional_stack[conditional_stack.length - 2].skip_section)) {
1509 : // condition true => process code within if
1510 16845 : conditional_stack[conditional_stack.length - 1].matched = true;
1511 16845 : conditional_stack[conditional_stack.length - 1].skip_section = false;
1512 : } else {
1513 : // skip lines until next preprocessing directive
1514 12144 : conditional_stack[conditional_stack.length - 1].skip_section = true;
1515 : }
1516 : }
1517 :
1518 32079 : void parse_pp_endif () {
1519 32079 : pp_eol ();
1520 :
1521 32079 : if (conditional_stack.length == 0) {
1522 1 : Report.error (get_source_reference (0), "syntax error, unexpected #endif");
1523 1 : return;
1524 : }
1525 :
1526 32078 : conditional_stack.length--;
1527 : }
1528 :
1529 32140 : bool parse_pp_symbol () {
1530 : int len = 0;
1531 311470 : while (current < end && is_ident_char (current[0])) {
1532 279330 : current++;
1533 279330 : column++;
1534 279330 : len++;
1535 : }
1536 :
1537 32140 : if (len == 0) {
1538 0 : Report.error (get_source_reference (0), "syntax error, expected identifier");
1539 0 : return false;
1540 : }
1541 :
1542 32140 : string identifier = ((string) (current - len)).substring (0, len);
1543 : bool defined;
1544 32140 : if (identifier == "true") {
1545 : defined = true;
1546 32140 : } else if (identifier == "false") {
1547 : defined = false;
1548 : } else {
1549 32140 : defined = source_file.context.is_defined (identifier);
1550 : }
1551 :
1552 32140 : return defined;
1553 : }
1554 :
1555 32143 : bool parse_pp_primary_expression () {
1556 32143 : if (current >= end) {
1557 0 : Report.error (get_source_reference (0), "syntax error, expected identifier");
1558 32143 : } else if (is_ident_char (current[0])) {
1559 32140 : return parse_pp_symbol ();
1560 3 : } else if (current[0] == '(') {
1561 3 : current++;
1562 3 : column++;
1563 3 : pp_space ();
1564 3 : bool result = parse_pp_expression ();
1565 3 : pp_space ();
1566 3 : if (current < end && current[0] == ')') {
1567 2 : current++;
1568 2 : column++;
1569 : } else {
1570 1 : Report.error (get_source_reference (0), "syntax error, expected `)'");
1571 : }
1572 3 : return result;
1573 : } else {
1574 0 : Report.error (get_source_reference (0), "syntax error, expected identifier");
1575 : }
1576 32143 : return false;
1577 : }
1578 :
1579 32194 : bool parse_pp_unary_expression () {
1580 32194 : if (current < end && current[0] == '!') {
1581 51 : current++;
1582 51 : column++;
1583 51 : pp_space ();
1584 51 : return !parse_pp_unary_expression ();
1585 : }
1586 :
1587 32143 : return parse_pp_primary_expression ();
1588 : }
1589 :
1590 32141 : bool parse_pp_equality_expression () {
1591 32141 : bool left = parse_pp_unary_expression ();
1592 32141 : pp_space ();
1593 32143 : while (true) {
1594 32143 : if (current < end - 1 && current[0] == '=' && current[1] == '=') {
1595 1 : current += 2;
1596 1 : column += 2;
1597 1 : pp_space ();
1598 1 : bool right = parse_pp_unary_expression ();
1599 1 : left = (left == right);
1600 32142 : } else if (current < end - 1 && current[0] == '!' && current[1] == '=') {
1601 1 : current += 2;
1602 1 : column += 2;
1603 1 : pp_space ();
1604 1 : bool right = parse_pp_unary_expression ();
1605 1 : left = (left != right);
1606 : } else {
1607 : break;
1608 : }
1609 : }
1610 : return left;
1611 : }
1612 :
1613 32137 : bool parse_pp_and_expression () {
1614 32137 : bool left = parse_pp_equality_expression ();
1615 32137 : pp_space ();
1616 32141 : while (current < end - 1 && current[0] == '&' && current[1] == '&') {
1617 4 : current += 2;
1618 4 : column += 2;
1619 4 : pp_space ();
1620 4 : bool right = parse_pp_equality_expression ();
1621 4 : left = left && right;
1622 : }
1623 : return left;
1624 : }
1625 :
1626 32136 : bool parse_pp_or_expression () {
1627 32136 : bool left = parse_pp_and_expression ();
1628 32136 : pp_space ();
1629 32137 : while (current < end - 1 && current[0] == '|' && current[1] == '|') {
1630 1 : current += 2;
1631 1 : column += 2;
1632 1 : pp_space ();
1633 1 : bool right = parse_pp_and_expression ();
1634 1 : left = left || right;
1635 : }
1636 : return left;
1637 : }
1638 :
1639 32136 : bool parse_pp_expression () {
1640 32136 : return parse_pp_or_expression ();
1641 : }
1642 :
1643 283233314 : bool whitespace () {
1644 283233314 : bool found = false;
1645 283233314 : bool bol = (column == 1);
1646 424287980 : while (current < end && current[0].isspace ()) {
1647 141054666 : if (current[0] == '\n') {
1648 21296941 : line++;
1649 21296941 : column = 0;
1650 21296941 : bol = true;
1651 : }
1652 141054666 : found = true;
1653 141054666 : current++;
1654 141054666 : column++;
1655 : }
1656 283233314 : if (bol && current < end && current[0] == '#') {
1657 93203 : pp_directive ();
1658 93203 : return true;
1659 : }
1660 283233314 : return found;
1661 : }
1662 :
1663 180204249 : bool comment (bool file_comment = false) {
1664 180204249 : if (current == null
1665 180204249 : || current > end - 2
1666 180197632 : || current[0] != '/'
1667 264926 : || (current[1] != '/' && current[1] != '*')) {
1668 179939388 : return false;
1669 : }
1670 :
1671 274766 : if (current[1] == '/') {
1672 23960 : SourceReference source_reference = null;
1673 23960 : if (file_comment) {
1674 6 : source_reference = get_source_reference (0);
1675 : }
1676 :
1677 : // single-line comment
1678 23960 : current += 2;
1679 23960 : char* begin = current;
1680 :
1681 : // skip until end of line or end of file
1682 1016250 : while (current < end && current[0] != '\n') {
1683 992290 : current++;
1684 : }
1685 :
1686 23960 : if (source_reference != null) {
1687 6 : push_comment (((string) begin).substring (0, (long) (current - begin)), source_reference, file_comment);
1688 : }
1689 : } else {
1690 240919 : SourceReference source_reference = null;
1691 :
1692 240919 : if (file_comment && current[2] == '*') {
1693 179939388 : return false;
1694 : }
1695 :
1696 240901 : if (current[2] == '*' || file_comment) {
1697 9888 : source_reference = get_source_reference (0);
1698 : }
1699 :
1700 240901 : current += 2;
1701 240901 : column += 2;
1702 :
1703 240901 : char* begin = current;
1704 7875370 : while (current < end - 1
1705 7875369 : && (current[0] != '*' || current[1] != '/')) {
1706 7634469 : if (current[0] == '\n') {
1707 109750 : line++;
1708 109750 : column = 0;
1709 : }
1710 7634469 : current++;
1711 7634469 : column++;
1712 : }
1713 :
1714 240901 : if (current == end - 1) {
1715 1 : Report.error (get_source_reference (0), "syntax error, expected */");
1716 1 : return true;
1717 : }
1718 :
1719 240900 : if (source_reference != null) {
1720 9887 : push_comment (((string) begin).substring (0, (long) (current - begin)), source_reference, file_comment);
1721 : }
1722 :
1723 240900 : current += 2;
1724 240900 : column += 2;
1725 : }
1726 :
1727 180204249 : return true;
1728 : }
1729 :
1730 179617767 : void space () {
1731 283216181 : while (whitespace () || comment ()) {
1732 : }
1733 : }
1734 :
1735 6605 : public void parse_file_comments () {
1736 17133 : while (whitespace () || comment (true)) {
1737 : }
1738 : }
1739 :
1740 9893 : void push_comment (string comment_item, SourceReference source_reference, bool file_comment) {
1741 9893 : if (comment_item[0] == '*') {
1742 4661 : if (_comment != null) {
1743 : // extra doc comment, add it to source file comments
1744 0 : source_file.add_comment (_comment);
1745 : }
1746 4661 : _comment = new Comment (comment_item, source_reference);
1747 : }
1748 :
1749 9893 : if (file_comment) {
1750 5232 : source_file.add_comment (new Comment (comment_item, source_reference));
1751 5232 : _comment = null;
1752 : }
1753 : }
1754 :
1755 : /**
1756 : * Clears and returns the content of the comment stack.
1757 : *
1758 : * @return saved comment
1759 : */
1760 12942471 : public Comment? pop_comment () {
1761 12942471 : if (_comment == null) {
1762 12937810 : return null;
1763 : }
1764 :
1765 4661 : var comment = _comment;
1766 4661 : _comment = null;
1767 4661 : return comment;
1768 : }
1769 : }
1770 :
|
