Skip to content
← Back

src/lexical/tokenizer.ghul

1
namespace Lexical is
2
use System.Exception
3
4
use Logging
5
use Source
6
7
8
use Ghul.Pipes
9
10
get_stack_trace() -> string is
11
try
12
return Logging.CALL_STACK.text
13
.split(['\n']) |>
14
skip(2) |>
15
take(5) |>
16
map(s => s.trim().replace("at ", "")) |> join(" ")
17
catch e: System.Exception
18
return "unknown"
19
yrt
20
si
21
22
@suppress("non-pascal-case-name")
23
class TOKENIZER_EXCEPTION(s: string): Exception is
24
super(s)
25
si
26
27
class TOKEN_MAP is
28
_map: Collections.MAP[string,TOKEN]
29
30
init() is
31
super.init()
32
_map = Collections.MAP[string,TOKEN](223)
33
si
34
35
[s: string]: TOKEN public =>
36
if _map.contains_key(s) then
37
_map[s]
38
else
39
TOKEN.IDENTIFIER
40
fi,
41
= t is
42
_map[s] = t
43
si
44
si
45
46
class TOKEN_PAIR(token: TOKEN, location: LOCATION, value: string init) is
47
value_string: string
48
49
// True when this token is the first on its source line. Stamped by
50
// TOKEN_LOOKAHEAD as tokens are first read, so it survives
51
// speculation replays.
52
first_on_line: bool public
53
54
// True when the token before this one closed a string literal.
55
// Stamped alongside first_on_line. A `;` carrying this, followed
56
// by another string, is separating fragments that would otherwise
57
// chain into one literal, so it is never redundant.
58
follows_string: bool public
59
60
name: string => TOKEN_NAMES[token]
61
62
to_string() -> string =>
63
"{location}: {TOKEN_NAMES[token]} {if value_string.length > 0 then "'{value_string}'" else "" fi}"
64
65
to_short_string() -> string =>
66
"{if value_string.length > 0 /\ value_string !~ TOKEN_NAMES[token] then "{TOKEN_NAMES[token]} \"{value_string}\"" else "'{TOKEN_NAMES[token]}'" fi}"
67
68
init(..) is
69
value_string = value
70
si
71
si
72
73
class TOKENIZER: TokenSource is
74
_prev_count: int
75
76
// Where a literal the input ended inside was reported: a string,
77
// a character literal or an interpolation format with no closing
78
// quote before end of input, which more input would close.
79
end_of_input_error_locations: Collections.LIST[LOCATION]
80
81
_logger: Logger
82
_token_pair: TOKEN_PAIR
83
_symbol_tokens: TOKEN_MAP static
84
_operator_chars: Collections.SET[char] static
85
_operator_tokens: Collections.MAP[string,TOKEN] static
86
_input: IO.TextReader
87
_end_of_file: bool
88
_prev_char: char
89
90
// Whether the last character read was the space that stands for the
91
// end of the input, which moved the cursor nowhere.
92
_read_end_of_file: bool
93
_cursor: LOCATION_CURSOR
94
95
_interpolation_depth: int
96
97
// How many braces are open in the code at each level of string
98
// interpolation, outermost first, so a `}` closes a brace the code
99
// opened before it ends an interpolation. Always one entry more
100
// than the interpolation depth.
101
_open_braces: Collections.LIST[int]
102
_expect_format_string: bool
103
104
_trivia: Collections.LIST[TRIVIA]
105
106
// The line the most recent token or comment started on, and whether
107
// the item being read is the first thing on its line. A `///` comment
108
// is a doc comment only when nothing precedes it on its line.
109
_content_line: int
110
_item_starts_line: bool
111
112
// Comments and blank-line markers, in source order. Discarded by the
113
// parser; consumed by the formatter to round-trip non-token content.
114
trivia: Collections.Iterable[TRIVIA] => _trivia
115
116
init() static is
117
_operator_chars = Collections.SET[char]([
118
'!', '$', '%', '^', '&', '*', '-', '+', '=', '|', ':', '@', '~', '#', '\\', '<', '>', '.', '?', '/'
119
]: char )
120
121
_operator_tokens = Collections.MAP[string,TOKEN]()
122
_operator_tokens["="] = TOKEN.ASSIGN
123
_operator_tokens[":"] = TOKEN.COLON
124
_operator_tokens["."] = TOKEN.DOT
125
_operator_tokens["->"] = TOKEN.ARROW_THIN
126
_operator_tokens["=>"] = TOKEN.ARROW_FAT
127
_operator_tokens["|>"] = TOKEN.BAR_ARROW
128
_operator_tokens["~>"] = TOKEN.TILDE_ARROW
129
_operator_tokens["?"] = TOKEN.QUESTION
130
_operator_tokens["@"] = TOKEN.AT
131
_operator_tokens["@@"] = TOKEN.AT_AT
132
_symbol_tokens = TOKEN_MAP()
133
_symbol_tokens["abstract"] = TOKEN.ABSTRACT
134
_symbol_tokens["namespace"] = TOKEN.NAMESPACE
135
_symbol_tokens["class"] = TOKEN.CLASS
136
_symbol_tokens["struct"] = TOKEN.STRUCT
137
_symbol_tokens["union"] = TOKEN.UNION
138
_symbol_tokens["partial"] = TOKEN.PARTIAL
139
_symbol_tokens["impl"] = TOKEN.IMPL
140
_symbol_tokens["enum"] = TOKEN.ENUM
141
_symbol_tokens["public"] = TOKEN.PUBLIC
142
_symbol_tokens["protected"] = TOKEN.PROTECTED
143
_symbol_tokens["private"] = TOKEN.PRIVATE
144
_symbol_tokens["field"] = TOKEN.FIELD
145
_symbol_tokens["static"] = TOKEN.STATIC
146
_symbol_tokens["innate"] = TOKEN.INNATE
147
_symbol_tokens["rec"] = TOKEN.REC
148
_symbol_tokens["if"] = TOKEN.IF
149
_symbol_tokens["else"] = TOKEN.ELSE
150
_symbol_tokens["while"] = TOKEN.WHILE
151
_symbol_tokens["do"] = TOKEN.DO
152
_symbol_tokens["for"] = TOKEN.FOR
153
_symbol_tokens["in"] = TOKEN.IN
154
_symbol_tokens["case"] = TOKEN.CASE
155
_symbol_tokens["when"] = TOKEN.WHEN
156
_symbol_tokens["default"] = TOKEN.DEFAULT
157
_symbol_tokens["break"] = TOKEN.BREAK
158
_symbol_tokens["continue"] = TOKEN.CONTINUE
159
_symbol_tokens["ref"] = TOKEN.REF
160
_symbol_tokens["ptr"] = TOKEN.PTR
161
_symbol_tokens["new"] = TOKEN.NEW
162
_symbol_tokens["throw"] = TOKEN.THROW
163
_symbol_tokens["return"] = TOKEN.RETURN
164
_symbol_tokens["cast"] = TOKEN.CAST
165
_symbol_tokens["try"] = TOKEN.TRY
166
_symbol_tokens["let"] = TOKEN.LET
167
_symbol_tokens["mut"] = TOKEN.MUT
168
_symbol_tokens["await"] = TOKEN.AWAIT
169
_symbol_tokens["catch"] = TOKEN.CATCH
170
_symbol_tokens["finally"] = TOKEN.FINALLY
171
_symbol_tokens["self"] = TOKEN.SELF
172
_symbol_tokens["super"] = TOKEN.SUPER
173
_symbol_tokens["null"] = TOKEN.NULL
174
_symbol_tokens["use"] = TOKEN.USE
175
_symbol_tokens["trait"] = TOKEN.TRAIT
176
_symbol_tokens["isa"] = TOKEN.ISA
177
_symbol_tokens["typeof"] = TOKEN.TYPEOF
178
// _symbol_tokens["operator"] = TOKEN.OPERATOR;
179
_symbol_tokens["is"] = TOKEN.IS
180
_symbol_tokens["si"] = TOKEN.SI
181
_symbol_tokens["then"] = TOKEN.THEN
182
_symbol_tokens["elif"] = TOKEN.ELIF
183
_symbol_tokens["fi"] = TOKEN.FI
184
_symbol_tokens["esac"] = TOKEN.ESAC
185
_symbol_tokens["lav"] = TOKEN.LAV
186
_symbol_tokens["val"] = TOKEN.VAL
187
_symbol_tokens["od"] = TOKEN.OD
188
_symbol_tokens["yield"] = TOKEN.YIELD
189
_symbol_tokens["yrt"] = TOKEN.YRT
190
_symbol_tokens["true"] = TOKEN.TRUE
191
_symbol_tokens["false"] = TOKEN.FALSE
192
_symbol_tokens["assert"] = TOKEN.ASSERT
193
si
194
195
// True when a bare occurrence of `name` would tokenise as a keyword
196
// rather than an identifier, so writing it back out unadorned would
197
// change its meaning. Such a name can only have reached the parser as
198
// an identifier via a backtick escape.
199
is_reserved_word(name: string?) -> bool static is
200
return name? /\ _symbol_tokens[name] != TOKEN.IDENTIFIER
201
si
202
203
// True when a bare occurrence of `name` would tokenise as a numeric
204
// literal rather than an identifier: it is composed only of identifier
205
// characters but begins with a digit. Such a name reached the parser
206
// as an identifier via a backtick escape.
207
is_numeric_identifier(name: string) -> bool static is
208
if name.length == 0 then
209
return false
210
fi
211
212
let first = name[0]
213
214
if !(first >= '0' /\ first <= '9') then
215
return false
216
fi
217
218
for c in name do
219
if !is_identifier_part(c) then
220
return false
221
fi
222
od
223
224
return true
225
si
226
227
// True when `name` is composed entirely of operator characters, so a
228
// bare occurrence would tokenise as an operator and a backtick escape
229
// is needed to refer to it as a plain identifier.
230
is_operator_name(name: string) -> bool static is
231
if name.length == 0 then
232
return false
233
fi
234
235
for c in name do
236
if !is_operator_char(c) then
237
return false
238
fi
239
od
240
241
return true
242
si
243
244
// An operator character is one of the ASCII set above, or any
245
// non-ASCII character Unicode classes as a symbol. A letter is
246
// never a symbol, so no character is both an operator character
247
// and an identifier character.
248
is_operator_char(c: char) -> bool static =>
249
_operator_chars.contains(c) \/
250
(cast int(c) > 0x7E /\ char.is_symbol(c))
251
252
// Identifier characters follow the categories C# draws on, less
253
// the format characters: those render as nothing, so admitting
254
// them would let two names that read alike be different names.
255
// Only the basic plane is covered - a character above it arrives
256
// as a surrogate pair, which this scanner reads as two characters.
257
is_identifier_start(c: char) -> bool static is
258
if (c >= 'a' /\ c <= 'z') \/ (c >= 'A' /\ c <= 'Z') \/ c == '_' then
259
return true
260
fi
261
262
if cast int(c) <= 0x7E then
263
return false
264
fi
265
266
return char.is_letter(c) \/ _is_letter_number(c)
267
si
268
269
is_identifier_part(c: char) -> bool static is
270
if (c >= '0' /\ c <= '9') \/ is_identifier_start(c) then
271
return true
272
fi
273
274
if cast int(c) <= 0x7E then
275
return false
276
fi
277
278
return _is_other_identifier_part(c)
279
si
280
281
is_format_char(c: char) -> bool static =>
282
cast int(c) > 0x7E /\ _is_format(c)
283
284
@IF.dotnet()
285
_is_letter_number(c: char) -> bool static =>
286
char.get_unicode_category(c) == System.Globalization.UnicodeCategory.LETTER_NUMBER
287
288
@IF.dotnet()
289
_is_other_identifier_part(c: char) -> bool static is
290
let category = char.get_unicode_category(c)
291
292
return category == System.Globalization.UnicodeCategory.DECIMAL_DIGIT_NUMBER \/
293
category == System.Globalization.UnicodeCategory.NON_SPACING_MARK \/
294
category == System.Globalization.UnicodeCategory.SPACING_COMBINING_MARK \/
295
category == System.Globalization.UnicodeCategory.CONNECTOR_PUNCTUATION
296
si
297
298
@IF.dotnet()
299
_is_format(c: char) -> bool static =>
300
char.get_unicode_category(c) == System.Globalization.UnicodeCategory.FORMAT
301
302
// Where the core library classifies only the scripts it covers, no
303
// other character is a letter number, a combining mark or a
304
// connector.
305
@IF.not.dotnet()
306
_is_letter_number(c: char) -> bool static => false
307
308
@IF.not.dotnet()
309
_is_other_identifier_part(c: char) -> bool static => false
310
311
// The format characters of the basic plane, each named by code so
312
// that none can slip into an identifier unseen.
313
@IF.not.dotnet()
314
_is_format(c: char) -> bool static is
315
let code = cast int(c)
316
317
return
318
code == 0xAD \/ code >= 0x600 /\ code <= 0x605 \/ code == 0x61C \/ code == 0x6DD \/
319
code == 0x70F \/ code == 0x180E \/ code >= 0x200B /\ code <= 0x200F \/
320
code >= 0x202A /\ code <= 0x202E \/ code >= 0x2060 /\ code <= 0x2064 \/
321
code >= 0x2066 /\ code <= 0x206F \/ code == 0xFEFF \/ code >= 0xFFF9 /\ code <= 0xFFFB
322
si
323
324
init(logger: Logger, file_name: string, i: IO.TextReader, is_internal_file: bool) is
325
super.init()
326
327
_logger = logger
328
329
_end_of_file = false
330
_open_braces = Collections.LIST[int]([0])
331
end_of_input_error_locations = Collections.LIST[LOCATION]()
332
_input = i
333
334
_trivia = Collections.LIST[TRIVIA]()
335
336
if is_internal_file then
337
_cursor = INTERNAL_LOCATION_CURSOR()
338
else
339
_cursor = LOCATION_CURSOR(file_name)
340
fi
341
342
_token_pair = TOKEN_PAIR(TOKEN.UNKNOWN, location, "")
343
344
skip_shebang_line()
345
si
346
347
// A `#!...` line is only meaningful as the file's first two bytes -
348
// the kernel's own rule for an executable script - so this only ever
349
// runs once, from the constructor, before anything else is read.
350
// `#` and `!` are both operator characters, so without this the line
351
// would otherwise tokenise as a run of operators rather than being
352
// read as a directive to skip.
353
skip_shebang_line() is
354
if _input.peek() != cast int('#') then
355
return
356
fi
357
358
_cursor.start()
359
360
let shebang_location = location
361
362
next_char()
363
364
if _input.peek() != cast int('!') then
365
prev_char('#')
366
return
367
fi
368
369
let buffer = System.Text.StringBuilder("#")
370
let c: char mut
371
do
372
c = next_char()
373
if c != '\n' then
374
buffer.append(c)
375
fi
376
377
if !(!_end_of_file /\ c != '\n') then
378
break
379
fi
380
od
381
382
_trivia.add(
383
TRIVIA.SHEBANG(
384
buffer.to_string().trim_end(),
385
shebang_location
386
)
387
)
388
si
389
390
is_end_of_file: bool => _end_of_file
391
392
location: LOCATION => _cursor.location
393
394
character_location: LOCATION => _cursor.character_location
395
396
advance_cursor(c: char) is
397
if c == cast char(13) then
398
return
399
fi
400
401
_cursor.next_column()
402
403
if c == '\n' then
404
_cursor.next_line()
405
fi
406
si
407
408
next_char() -> char is
409
let c: char mut
410
411
_read_end_of_file = false
412
413
if _prev_char != cast char(0) then
414
c = _prev_char
415
advance_cursor(c)
416
_prev_char = cast char(0)
417
return c
418
fi
419
420
if _end_of_file then
421
_read_end_of_file = true
422
return ' '
423
fi
424
425
let c0: int mut = _input.read()
426
427
if c0 == 13 then
428
c0 = 32
429
fi
430
431
if c0 == -1 \/ c0 == 12 then
432
_end_of_file = true
433
_read_end_of_file = true
434
return ' '
435
fi
436
437
c = cast char(c0)
438
439
_cursor.save()
440
advance_cursor(c)
441
442
return c
443
si
444
445
// Pushing back the end of the input leaves everything as it is: it
446
// moved the cursor nowhere, and reading again gives it again.
447
// Restoring would move the cursor back over the character before
448
// it, cutting the input's last token short by one.
449
prev_char(c: char) is
450
if _read_end_of_file then
451
return
452
fi
453
454
_cursor.restore()
455
_prev_char = c
456
si
457
458
current_string: string =>
459
_token_pair.value_string
460
461
read_escape() -> char is
462
let c: char mut = next_char()
463
let result: int mut = 0
464
if c == 't' then
465
return cast char(9)
466
elif c == 'n' then
467
return '\n'
468
elif c == 'r' then
469
return cast char(13)
470
elif c == '\\' then
471
return '\\'
472
elif c == 'u' then
473
// Exactly four hex digits, which is exactly one UTF-16 code
474
// unit and so exactly one char. Fixed width so that the
475
// escape never consumes text that follows it: the characters
476
// after a code are letters as often as not, and a-f are
477
// letters.
478
let escape_location = character_location
479
480
for i in 0..4 do
481
c = next_char()
482
483
if c >= '0' /\ c <= '9' then
484
result = 16 * result + cast int(c - '0')
485
elif c >= 'a' /\ c <= 'f' then
486
result = 16 * result + cast int(c - 'a') + 10
487
elif c >= 'A' /\ c <= 'F' then
488
result = 16 * result + cast int(c - 'A') + 10
489
else
490
prev_char(c)
491
_logger.lexer_error(escape_location, "expected four hex digits after \\u")
492
return cast char(0)
493
fi
494
od
495
496
return cast char(result)
497
elif c>='0' /\ c<='7' then
498
_logger.warn(character_location, "deprecated-octal-escape", "octal escape is deprecated - write the character as \\uXXXX")
499
500
while c >= '0' /\ c <= '7' do
501
result = 8 * result + cast int(c - '0')
502
c = next_char()
503
od
504
505
prev_char(c)
506
return cast char(result)
507
else
508
return c
509
fi
510
si
511
512
skip_white_space() -> char is
513
let c: char mut = _
514
let newlines mut = 0
515
let blank_location: LOCATION? mut = null
516
do
517
c = next_char()
518
519
if c == '\n' then
520
newlines = newlines + 1
521
if newlines == 2 then
522
blank_location = character_location
523
fi
524
fi
525
526
let is_white_space = (c==' ' \/ c==cast char(9) \/ c=='\n')
527
528
let should_break = is_end_of_file \/ !is_white_space
529
530
if is_end_of_file \/ !is_white_space then
531
break
532
fi
533
od
534
535
if newlines >= 2 /\ blank_location? then
536
_trivia.add(TRIVIA.BLANK_LINE(blank_location))
537
fi
538
539
return c
540
si
541
542
expect_format_specifier() is
543
_expect_format_string = true
544
si
545
546
read_token() -> TOKEN_PAIR is
547
let r: TOKEN mut
548
549
let c mut = skip_white_space()
550
551
if _end_of_file then
552
return TOKEN_PAIR(TOKEN.END_OF_INPUT, location, "")
553
fi
554
555
_cursor.start()
556
557
_note_item_start(location.start_line)
558
559
let _buffer mut = System.Text.StringBuilder()
560
561
if _expect_format_string then
562
_expect_format_string = false
563
564
_buffer = System.Text.StringBuilder()
565
566
while !_end_of_file /\ c != '\n' /\ c != '}' /\ c != '"' do
567
_buffer.append(c)
568
569
c = next_char()
570
od
571
572
if c == '}' then
573
prev_char(c)
574
elif c == '\n' then
575
prev_char(c)
576
_logger.lexer_error(character_location, "newline in interpolation format string")
577
return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
578
elif c == '"' then
579
_logger.lexer_error(character_location, "expected '}}' after format string")
580
return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
581
elif _end_of_file then
582
_logger.lexer_error(character_location, "end of file in interpolation format string")
583
end_of_input_error_locations.add(character_location)
584
return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
585
fi
586
587
return TOKEN_PAIR(TOKEN.FORMAT_STRING, location, _buffer.to_string())
588
589
elif c >= '0' /\ c <= '9' then
590
let is_float mut = false
591
592
_buffer = System.Text.StringBuilder()
593
_buffer.append(c)
594
c = next_char()
595
596
if c=='x' \/ c=='X' then
597
_buffer.append(c)
598
c = next_char()
599
while
600
(c>='0'/\c<='9') \/ (c>='A'/\c<='F') \/ (c>='a'/\c<='f') \/ c == '_'
601
do
602
if c != '_' then
603
_buffer.append(c)
604
fi
605
c = next_char()
606
od
607
else
608
let seen_dot mut = false
609
let pc: char mut = _
610
611
while
612
(c >= '0' /\ c <= '9') \/ c == '.' \/ c == '_'
613
do
614
if c == '.' then
615
if pc == '.' then
616
is_float = false
617
618
// bodge: can only push back one char, so use Unicode '‥' to signal we actually want to push back '..'
619
c = '‥'
620
break
621
elif seen_dot then
622
break
623
else
624
is_float = true
625
seen_dot = true
626
fi
627
else
628
if pc == '.' then
629
_buffer.append('.')
630
fi
631
632
if c != '_' then
633
_buffer.append(c)
634
fi
635
fi
636
637
pc = c
638
c = next_char()
639
od
640
fi
641
642
if is_float then
643
if c == 'e' \/ c == 'E' then
644
_buffer.append(c)
645
c = next_char()
646
647
if c == '-' then
648
_buffer.append(c)
649
c = next_char()
650
fi
651
652
if c >= '0' \/ c <= '9' \/ c == '_' then
653
while (c >= '0' /\ c <= '9') \/ c == '_' do
654
if c != '_' then
655
_buffer.append(c)
656
fi
657
658
c = next_char()
659
od
660
else
661
_logger.lexer_error(location, "expected exponent in float literal")
662
fi
663
fi
664
665
if c == 's' \/ c == 'S' \/ c == 'd' \/ c == 'D' \/ c == 'm' \/ c == 'M' then
666
_buffer.append(c)
667
c = next_char()
668
fi
669
else
670
if c == 'm' \/ c == 'M' then
671
_buffer.append(c)
672
c = next_char()
673
is_float = true
674
elif c == '`' then
675
// A backtick attaches a type suffix that would
676
// otherwise be read as a digit of the literal's own
677
// radix - `b` and `c` are hex digits as well as size
678
// selectors, so in hex they are only ever reachable
679
// this way. The backtick stays in the token text: it
680
// is what tells the classifier the suffix is explicit.
681
_buffer.append(c)
682
c = next_char()
683
684
let suffix_start = _buffer.length
685
686
if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then
687
_buffer.append(c)
688
c = next_char()
689
fi
690
691
if "bBcCsSiIlLnNwW".contains(c) then
692
_buffer.append(c)
693
c = next_char()
694
fi
695
696
if _buffer.length == suffix_start then
697
_logger.lexer_error(character_location, "expected a type suffix after ` in numeric literal")
698
fi
699
else
700
if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then
701
_buffer.append(c)
702
c = next_char()
703
fi
704
705
if "bBcCsSiIlLnNwW".contains(c) then
706
_buffer.append(c)
707
c = next_char()
708
fi
709
fi
710
fi
711
712
prev_char(c)
713
714
if is_float then
715
return TOKEN_PAIR(TOKEN.FLOAT_LITERAL, location, _buffer.to_string())
716
else
717
return TOKEN_PAIR(TOKEN.INT_LITERAL, location, _buffer.to_string())
718
fi
719
720
elif is_identifier_start(c) \/ c == '`' then
721
let was_escaped: bool mut = false
722
723
if c == '`' then
724
c = next_char()
725
726
if c == '[' then
727
return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "")
728
elif is_operator_char(c) then
729
let o = read_operator(c)
730
return TOKEN_PAIR(TOKEN.IDENTIFIER, o.location, o.value_string)
731
else
732
was_escaped = true
733
fi
734
fi
735
736
_buffer = System.Text.StringBuilder()
737
738
while is_identifier_part(c) \/ is_format_char(c) do
739
if is_format_char(c) then
740
_logger.lexer_error(
741
character_location,
742
"format character U+{cast int(c):X4} in identifier")
743
else
744
_buffer.append(c)
745
fi
746
747
c = next_char()
748
od
749
750
prev_char(c)
751
let s = _buffer.to_string()
752
753
if was_escaped then
754
if s.length == 0 then
755
_logger.lexer_error(location, "expected an identifier after backtick")
756
fi
757
758
r = TOKEN.IDENTIFIER
759
else
760
r = _symbol_tokens[s]
761
fi
762
763
return TOKEN_PAIR(r, location, s)
764
elif c == '/' then
765
let comment_location = location
766
c = next_char()
767
if c == '/' then
768
read_line_comment(comment_location)
769
770
return read_token()
771
elif c == '*' then
772
read_block_comment(comment_location)
773
774
return read_token()
775
else
776
prev_char(c)
777
return read_operator('/')
778
fi
779
elif is_operator_char(c) then
780
return read_operator(c)
781
fi
782
783
case c
784
785
when '\'' then
786
_buffer = System.Text.StringBuilder()
787
788
c = next_char()
789
while c != cast char(39) do
790
if c == cast char(92) then
791
c = read_escape()
792
_buffer.append(c)
793
c = next_char()
794
else
795
_buffer.append(c)
796
c = next_char()
797
fi
798
799
if _end_of_file then
800
_logger.lexer_error(location, "end of file in character literal")
801
end_of_input_error_locations.add(location)
802
break
803
fi
804
od
805
806
if _buffer.length < 1 then
807
_logger.lexer_error(location, "zero length character literal")
808
elif _buffer.length > 1 then
809
_logger.lexer_error(location, "character literal is too long")
810
fi
811
return TOKEN_PAIR(TOKEN.CHAR_LITERAL, location, _buffer.to_string())
812
813
when '(' then
814
return TOKEN_PAIR(TOKEN.PAREN_OPEN, location, "")
815
816
when ')' then
817
return TOKEN_PAIR(TOKEN.PAREN_CLOSE, location, "")
818
819
when '[' then
820
c = next_char()
821
if _end_of_file then
822
return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "")
823
elif c == ']' then
824
return TOKEN_PAIR(TOKEN.ARRAY_DEF, location, "")
825
else
826
prev_char(c)
827
return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "")
828
fi
829
830
when ']' then
831
return TOKEN_PAIR(TOKEN.SQUARE_CLOSE, location, "")
832
833
when ',' then
834
return TOKEN_PAIR(TOKEN.COMMA, location, "")
835
836
when ';' then
837
return TOKEN_PAIR(TOKEN.SEMICOLON, location, "")
838
839
when '‥' then
840
return read_operator(System.Text.StringBuilder(".."), next_char())
841
842
when '⟦' then
843
return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "")
844
845
when '"' then
846
return string_enter()
847
848
when '{' then
849
return _open_brace()
850
851
when '}' then
852
return _close_brace_or_exit_interpolation()
853
854
else
855
return TOKEN_PAIR(TOKEN.UNKNOWN, location, "{c}")
856
esac
857
si
858
859
read_operator(c: char mut) -> TOKEN_PAIR =>
860
read_operator(System.Text.StringBuilder(), c)
861
862
// `seed` carries the characters the caller has already consumed on
863
// this token's behalf. The numeric-literal scanner can push back only
864
// one character, so it hands a `..` following a number over as the
865
// single `‥`; seeding with the two dots lets the scan continue into
866
// whatever operator characters follow, so `1..<2` and `i..<2` produce
867
// the same token.
868
read_operator(seed: System.Text.StringBuilder, c: char mut) -> TOKEN_PAIR is
869
let _buffer = seed
870
let first = if _buffer.length > 0 then _buffer[0] else c fi
871
872
// A `.` immediately after a bare `!` or `?` ends the operator: it
873
// begins a member access (`x!.foo`, `x?.foo`), not a longer infix
874
// operator. Longer operators starting with `!`/`?` (e.g. `!=`) and
875
// dot operators (`..`) are unaffected.
876
while
877
is_operator_char(c) /\
878
!(_buffer.length == 1 /\ (first == '!' \/ first == '?') /\ c == '.')
879
do
880
_buffer.append(c)
881
c = next_char()
882
od
883
884
prev_char(c)
885
886
if _buffer.length < 1 then
887
return TOKEN_PAIR(TOKEN.FIRST, location, "")
888
fi
889
890
let s = _buffer.to_string()
891
892
let r mut = TOKEN.OPERATOR
893
894
if _operator_tokens.contains_key(s) then
895
r = _operator_tokens[s]
896
fi
897
898
return TOKEN_PAIR(r, location, s)
899
si
900
901
read_string_fragment() -> (fragment: string, c: char) is
902
let fragment = System.Text.StringBuilder()
903
904
let c mut = next_char()
905
906
while !_end_of_file /\ c != '\n' /\ c != '"' do
907
if c == cast char(92) then
908
c = read_escape()
909
fragment.append(c)
910
c = next_char()
911
elif c == '{' then
912
c = next_char()
913
914
if c == '{' then
915
fragment.append(c)
916
c = next_char()
917
else
918
prev_char(c)
919
c = '{'
920
921
break
922
fi
923
elif c == '}' then
924
c = next_char()
925
926
if c == '}' then
927
fragment.append(c)
928
c = next_char()
929
else
930
_logger.lexer_error(character_location, "unmatched '}}' in string interpolation")
931
932
fragment.append('}')
933
fi
934
else
935
fragment.append(c)
936
c = next_char()
937
fi
938
od
939
940
if c == '\n' then
941
_logger.lexer_error(character_location, "newline in string literal")
942
elif _end_of_file then
943
_logger.lexer_error(character_location, "end of file in string literal")
944
end_of_input_error_locations.add(character_location)
945
fi
946
947
return (fragment.to_string(), c)
948
si
949
950
_note_item_start(line: int) is
951
_item_starts_line = line != _content_line
952
_content_line = line
953
si
954
955
// Three slashes and not four: a longer run is a divider, not
956
// documentation.
957
_is_doc_comment(text: string) -> bool =>
958
_item_starts_line /\
959
text.starts_with("///") /\
960
(text.length == 3 \/ text[3] != '/')
961
962
// Consume the rest of a line comment whose `//` has been read,
963
// recording it as trivia.
964
read_line_comment(comment_location: LOCATION) is
965
let buffer = System.Text.StringBuilder("//")
966
let c: char mut
967
do
968
c = next_char()
969
if c != '\n' then
970
buffer.append(c)
971
fi
972
973
if !(!_end_of_file/\c!='\n') then
974
break
975
fi
976
od
977
978
let text = buffer.to_string().trim_end()
979
980
if _is_doc_comment(text) then
981
_trivia.add(TRIVIA.DOC_COMMENT(text, comment_location))
982
else
983
_trivia.add(TRIVIA.LINE_COMMENT(text, comment_location))
984
fi
985
si
986
987
// Consume the rest of a block comment whose `/*` has been read,
988
// recording it as trivia.
989
read_block_comment(comment_location: LOCATION) is
990
let buffer = System.Text.StringBuilder("/*")
991
let c: char mut
992
do
993
c = next_char()
994
995
if c == '*' then
996
c = next_char()
997
if c == '/' then
998
break
999
fi
1000
1001
buffer.append('*')
1002
fi
1003
1004
buffer.append(c)
1005
1006
if _end_of_file then
1007
break
1008
fi
1009
od
1010
1011
buffer.append("*/")
1012
1013
_trivia.add(
1014
TRIVIA.BLOCK_COMMENT(
1015
buffer.to_string(),
1016
comment_location
1017
)
1018
)
1019
si
1020
1021
// Adjacent string-literal concatenation: while a closed `"..."`
1022
// fragment is followed (after whitespace or comments) by another
1023
// `"`, fold the next fragment into this token. Stops at the first
1024
// fragment that doesn't close with `"` (i.e. starts an
1025
// interpolation or hits a newline). Comments between fragments are
1026
// recorded as trivia and chained across, so stripping them cannot
1027
// change what the token means; only a `;` separates two
1028
// otherwise-adjacent literals.
1029
//
1030
// We treat the whole `"..." "..." "..."` run as a single token.
1031
// The token's end position is wherever the final fragment
1032
// terminated — we snapshot the cursor's location after each
1033
// successful fragment close. When the chain fails (we consume
1034
// whitespace then see a non-`"` char), the cursor is past the
1035
// whitespace, but we return the snapshot taken at the previous
1036
// close so the token's location end isn't smeared across the
1037
// following whitespace. The stream stays at the non-`"` char;
1038
// the next read_token picks up from there normally.
1039
chain_adjacent_string_fragments(initial_fragment: string, initial_terminator: char) -> (fragment: string, c: char, loc: Source.LOCATION) is
1040
let buffer = System.Text.StringBuilder()
1041
buffer.append(initial_fragment)
1042
let c mut = initial_terminator
1043
let saved_loc mut = location
1044
1045
let done mut = false
1046
while !done /\ c == '"' do
1047
let peeked = skip_white_space()
1048
if peeked == '"' then
1049
let (next_fragment, next_terminator) = read_string_fragment()
1050
buffer.append(next_fragment)
1051
c = next_terminator
1052
saved_loc = location
1053
elif
1054
peeked == '/' /\
1055
(_input.peek() == cast int('/') \/ _input.peek() == cast int('*'))
1056
then
1057
// A comment counts as the whitespace it sits in, here as
1058
// everywhere. The raw peek is sound: the pushback slot is
1059
// always empty after skip_white_space hands a char out, so
1060
// the reader's next character is the one after the `/`.
1061
let comment_location = location
1062
1063
_note_item_start(comment_location.start_line)
1064
1065
if next_char() == '/' then
1066
read_line_comment(comment_location)
1067
else
1068
read_block_comment(comment_location)
1069
fi
1070
else
1071
prev_char(peeked)
1072
done = true
1073
fi
1074
od
1075
1076
return (buffer.to_string(), c, saved_loc)
1077
si
1078
1079
string_enter() -> TOKEN_PAIR is
1080
let (fragment, c) mut = read_string_fragment()
1081
1082
let loc: Source.LOCATION mut
1083
(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c)
1084
1085
// The closing quote settles this before anything else is
1086
// consulted: the adjacency lookahead peeks past a literal that
1087
// already closed, and can reach the end of the file doing it,
1088
// so the end-of-file flag says nothing about this literal once
1089
// its quote has been seen.
1090
if c == '"' then
1091
return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1092
elif c == '\n' \/ _end_of_file then
1093
_reset_interpolation()
1094
1095
return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment)
1096
elif c == '{' then
1097
_enter_interpolation()
1098
1099
return TOKEN_PAIR(TOKEN.ENTER_STRING, loc, fragment)
1100
else
1101
_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation")
1102
1103
return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1104
fi
1105
si
1106
1107
interpolation_clear() is
1108
_reset_interpolation()
1109
si
1110
1111
_enter_interpolation() is
1112
_interpolation_depth = _interpolation_depth + 1
1113
_open_braces.add(0)
1114
si
1115
1116
_leave_interpolation() is
1117
_interpolation_depth = _interpolation_depth - 1
1118
_open_braces.remove_at(_open_braces.count - 1)
1119
si
1120
1121
_reset_interpolation() is
1122
_interpolation_depth = 0
1123
_open_braces.clear()
1124
_open_braces.add(0)
1125
si
1126
1127
_open_brace() -> TOKEN_PAIR is
1128
_open_braces[_open_braces.count - 1] = _open_braces[_open_braces.count - 1] + 1
1129
1130
return TOKEN_PAIR(TOKEN.BRACE_OPEN, location, "")
1131
si
1132
1133
_close_brace_or_exit_interpolation() -> TOKEN_PAIR is
1134
let open = _open_braces[_open_braces.count - 1]
1135
1136
if open > 0 then
1137
_open_braces[_open_braces.count - 1] = open - 1
1138
1139
return TOKEN_PAIR(TOKEN.BRACE_CLOSE, location, "")
1140
fi
1141
1142
return interpolation_exit('}')
1143
si
1144
1145
interpolation_exit(c: char mut) -> TOKEN_PAIR is
1146
if _interpolation_depth == 0 then
1147
_logger.lexer_error(character_location, "unmatched '\"'")
1148
next_char()
1149
return read_token()
1150
fi
1151
1152
let fragment mut = ""
1153
1154
if c == '}' then
1155
(fragment, c) = read_string_fragment()
1156
fi
1157
1158
let loc: Source.LOCATION mut
1159
(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c)
1160
1161
// Closing quote first, for the same reason as in string_enter.
1162
if c == '"' then
1163
_leave_interpolation()
1164
1165
return TOKEN_PAIR(TOKEN.EXIT_STRING, loc, fragment)
1166
elif c == '\n' \/ _end_of_file then
1167
_reset_interpolation()
1168
return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment)
1169
elif c == '{' then
1170
return TOKEN_PAIR(TOKEN.CONTINUE_STRING, loc, fragment)
1171
else
1172
_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation")
1173
1174
return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1175
fi
1176
si
1177
si
1178
si