src/lexical/tokenizer.ghul

1		namespace Lexical is
2		use System.Exception
3
4		use Logging
5		use Source
6
7		use System.Globalization.UnicodeCategory
8
9		use Ghul.Pipes
10
11		get_stack_trace() -> string is
12		try
13		return System.Diagnostics.StackTrace()
14		.to_string()
15		.split(['\n']) \|>
16		skip(2) \|>
17		take(5) \|>
18		map(s => s.trim().replace("at ", "")) \|> join(" ")
19		catch e: System.Exception
20		return "unknown"
21		yrt
22		si
23
24		class TOKENIZER_EXCEPTION(s: string): Exception is
25		super(s)
26		si
27
28		class TOKEN_MAP is
29		_map: Collections.MAP[string,TOKEN]
30
31		init() is
32		super.init()
33		_map = Collections.MAP[string,TOKEN](223)
34		si
35
36		[s: string]: TOKEN public =>
37		if _map.contains_key(s) then
38		_map[s]
39		else
40		TOKEN.IDENTIFIER
41		fi,
42		= t is
43		_map[s] = t
44		si
45		si
46
47		class TOKEN_PAIR(token: TOKEN, location: LOCATION, value: string init) is
48		value_string: string
49
50		// True when this token is the first on its source line. Stamped by
51		// TOKEN_LOOKAHEAD as tokens are first read, so it survives
52		// speculation replays.
53		first_on_line: bool public
54
55		// True when the token before this one closed a string literal.
56		// Stamped alongside first_on_line. A `;` carrying this, followed
57		// by another string, is separating fragments that would otherwise
58		// chain into one literal, so it is never redundant.
59		follows_string: bool public
60
61		name: string => TOKEN_NAMES[token]
62
63		to_string() -> string =>
64		"{location}: {TOKEN_NAMES[token]} {if value_string.length > 0 then "'{value_string}'" else "" fi}"
65
66		to_short_string() -> string =>
67		"{if value_string.length > 0 /\ value_string !~ TOKEN_NAMES[token] then "{TOKEN_NAMES[token]} \"{value_string}\"" else "'{TOKEN_NAMES[token]}'" fi}"
68
69		init(..) is
70		value_string = value
71		si
72		si
73
74		class TOKENIZER: TokenSource is
75		_prev_count: int
76
77		// Where a literal the input ended inside was reported: a string,
78		// a character literal or an interpolation format with no closing
79		// quote before end of input, which more input would close.
80		end_of_input_error_locations: Collections.LIST[LOCATION]
81
82		_logger: Logger
83		_token_pair: TOKEN_PAIR
84		_symbol_tokens: TOKEN_MAP static
85		_operator_chars: Collections.SET[char] static
86		_operator_tokens: Collections.MAP[string,TOKEN] static
87		_input: IO.TextReader
88		_end_of_file: bool
89		_prev_char: char
90
91		// Whether the last character read was the space that stands for the
92		// end of the input, which moved the cursor nowhere.
93		_read_end_of_file: bool
94		_cursor: LOCATION_CURSOR
95
96		_interpolation_depth: int
97		_expect_format_string: bool
98
99		_trivia: Collections.LIST[TRIVIA]
100
101		// Comments and blank-line markers, in source order. Discarded by the
102		// parser; consumed by the formatter to round-trip non-token content.
103		trivia: Collections.Iterable[TRIVIA] => _trivia
104
105		init() static is
106		_operator_chars = Collections.SET[char]([
107		'!', '$', '%', '^', '&', '*', '-', '+', '=', '\|', ':', '@', '~', '#', '\\', '<', '>', '.', '?', '/'
108		]: char )
109
110		_operator_tokens = Collections.MAP[string,TOKEN]()
111		_operator_tokens["="] = TOKEN.ASSIGN
112		_operator_tokens[":"] = TOKEN.COLON
113		_operator_tokens["."] = TOKEN.DOT
114		_operator_tokens["->"] = TOKEN.ARROW_THIN
115		_operator_tokens["=>"] = TOKEN.ARROW_FAT
116		_operator_tokens["\|>"] = TOKEN.BAR_ARROW
117		_operator_tokens["~>"] = TOKEN.TILDE_ARROW
118		_operator_tokens["?"] = TOKEN.QUESTION
119		_operator_tokens["@"] = TOKEN.AT
120		_operator_tokens["@@"] = TOKEN.AT_AT
121		_symbol_tokens = TOKEN_MAP()
122		_symbol_tokens["abstract"] = TOKEN.ABSTRACT
123		_symbol_tokens["namespace"] = TOKEN.NAMESPACE
124		_symbol_tokens["class"] = TOKEN.CLASS
125		_symbol_tokens["struct"] = TOKEN.STRUCT
126		_symbol_tokens["union"] = TOKEN.UNION
127		_symbol_tokens["partial"] = TOKEN.PARTIAL
128		_symbol_tokens["impl"] = TOKEN.IMPL
129		_symbol_tokens["enum"] = TOKEN.ENUM
130		_symbol_tokens["public"] = TOKEN.PUBLIC
131		_symbol_tokens["protected"] = TOKEN.PROTECTED
132		_symbol_tokens["private"] = TOKEN.PRIVATE
133		_symbol_tokens["field"] = TOKEN.FIELD
134		_symbol_tokens["static"] = TOKEN.STATIC
135		_symbol_tokens["innate"] = TOKEN.INNATE
136		_symbol_tokens["rec"] = TOKEN.REC
137		_symbol_tokens["if"] = TOKEN.IF
138		_symbol_tokens["else"] = TOKEN.ELSE
139		_symbol_tokens["while"] = TOKEN.WHILE
140		_symbol_tokens["do"] = TOKEN.DO
141		_symbol_tokens["for"] = TOKEN.FOR
142		_symbol_tokens["in"] = TOKEN.IN
143		_symbol_tokens["case"] = TOKEN.CASE
144		_symbol_tokens["when"] = TOKEN.WHEN
145		_symbol_tokens["default"] = TOKEN.DEFAULT
146		_symbol_tokens["break"] = TOKEN.BREAK
147		_symbol_tokens["continue"] = TOKEN.CONTINUE
148		_symbol_tokens["ref"] = TOKEN.REF
149		_symbol_tokens["ptr"] = TOKEN.PTR
150		_symbol_tokens["new"] = TOKEN.NEW
151		_symbol_tokens["throw"] = TOKEN.THROW
152		_symbol_tokens["return"] = TOKEN.RETURN
153		_symbol_tokens["cast"] = TOKEN.CAST
154		_symbol_tokens["try"] = TOKEN.TRY
155		_symbol_tokens["let"] = TOKEN.LET
156		_symbol_tokens["mut"] = TOKEN.MUT
157		_symbol_tokens["await"] = TOKEN.AWAIT
158		_symbol_tokens["catch"] = TOKEN.CATCH
159		_symbol_tokens["finally"] = TOKEN.FINALLY
160		_symbol_tokens["self"] = TOKEN.SELF
161		_symbol_tokens["super"] = TOKEN.SUPER
162		_symbol_tokens["null"] = TOKEN.NULL
163		_symbol_tokens["use"] = TOKEN.USE
164		_symbol_tokens["trait"] = TOKEN.TRAIT
165		_symbol_tokens["isa"] = TOKEN.ISA
166		_symbol_tokens["typeof"] = TOKEN.TYPEOF
167		// _symbol_tokens["operator"] = TOKEN.OPERATOR;
168		_symbol_tokens["is"] = TOKEN.IS
169		_symbol_tokens["si"] = TOKEN.SI
170		_symbol_tokens["then"] = TOKEN.THEN
171		_symbol_tokens["elif"] = TOKEN.ELIF
172		_symbol_tokens["fi"] = TOKEN.FI
173		_symbol_tokens["esac"] = TOKEN.ESAC
174		_symbol_tokens["lav"] = TOKEN.LAV
175		_symbol_tokens["val"] = TOKEN.VAL
176		_symbol_tokens["od"] = TOKEN.OD
177		_symbol_tokens["yield"] = TOKEN.YIELD
178		_symbol_tokens["yrt"] = TOKEN.YRT
179		_symbol_tokens["true"] = TOKEN.TRUE
180		_symbol_tokens["false"] = TOKEN.FALSE
181		_symbol_tokens["assert"] = TOKEN.ASSERT
182		si
183
184		// True when a bare occurrence of `name` would tokenise as a keyword
185		// rather than an identifier, so writing it back out unadorned would
186		// change its meaning. Such a name can only have reached the parser as
187		// an identifier via a backtick escape.
188		is_reserved_word(name: string?) -> bool static is
189		return name? /\ _symbol_tokens[name] != TOKEN.IDENTIFIER
190		si
191
192		// True when a bare occurrence of `name` would tokenise as a numeric
193		// literal rather than an identifier: it is composed only of identifier
194		// characters but begins with a digit. Such a name reached the parser
195		// as an identifier via a backtick escape.
196		is_numeric_identifier(name: string) -> bool static is
197		if name.length == 0 then
198		return false
199		fi
200
201		let first = name[0]
202
203		if !(first >= '0' /\ first <= '9') then
204		return false
205		fi
206
207		for c in name do
208		if !is_identifier_part(c) then
209		return false
210		fi
211		od
212
213		return true
214		si
215
216		// True when `name` is composed entirely of operator characters, so a
217		// bare occurrence would tokenise as an operator and a backtick escape
218		// is needed to refer to it as a plain identifier.
219		is_operator_name(name: string) -> bool static is
220		if name.length == 0 then
221		return false
222		fi
223
224		for c in name do
225		if !is_operator_char(c) then
226		return false
227		fi
228		od
229
230		return true
231		si
232
233		// An operator character is one of the ASCII set above, or any
234		// non-ASCII character Unicode classes as a symbol. A letter is
235		// never a symbol, so no character is both an operator character
236		// and an identifier character.
237		is_operator_char(c: char) -> bool static =>
238		_operator_chars.contains(c) \/
239		(cast int(c) > 0x7E /\ char.is_symbol(c))
240
241		// Identifier characters follow the categories C# draws on, less
242		// the format characters: those render as nothing, so admitting
243		// them would let two names that read alike be different names.
244		// Only the basic plane is covered - a character above it arrives
245		// as a surrogate pair, which this scanner reads as two characters.
246		is_identifier_start(c: char) -> bool static is
247		if (c >= 'a' /\ c <= 'z') \/ (c >= 'A' /\ c <= 'Z') \/ c == '_' then
248		return true
249		fi
250
251		if cast int(c) <= 0x7E then
252		return false
253		fi
254
255		return char.is_letter(c) \/
256		char.get_unicode_category(c) == UnicodeCategory.LETTER_NUMBER
257		si
258
259		is_identifier_part(c: char) -> bool static is
260		if (c >= '0' /\ c <= '9') \/ is_identifier_start(c) then
261		return true
262		fi
263
264		if cast int(c) <= 0x7E then
265		return false
266		fi
267
268		let category = char.get_unicode_category(c)
269
270		return category == UnicodeCategory.DECIMAL_DIGIT_NUMBER \/
271		category == UnicodeCategory.NON_SPACING_MARK \/
272		category == UnicodeCategory.SPACING_COMBINING_MARK \/
273		category == UnicodeCategory.CONNECTOR_PUNCTUATION
274		si
275
276		is_format_char(c: char) -> bool static =>
277		cast int(c) > 0x7E /\
278		char.get_unicode_category(c) == UnicodeCategory.FORMAT
279
280		init(logger: Logger, file_name: string, i: IO.TextReader, is_internal_file: bool) is
281		super.init()
282
283		_logger = logger
284
285		_end_of_file = false
286		end_of_input_error_locations = Collections.LIST[LOCATION]()
287		_input = i
288
289		_trivia = Collections.LIST[TRIVIA]()
290
291		if is_internal_file then
292		_cursor = INTERNAL_LOCATION_CURSOR()
293		else
294		_cursor = LOCATION_CURSOR(file_name)
295		fi
296
297		_token_pair = TOKEN_PAIR(TOKEN.UNKNOWN, location, "")
298
299		skip_shebang_line()
300		si
301
302		// A `#!...` line is only meaningful as the file's first two bytes -
303		// the kernel's own rule for an executable script - so this only ever
304		// runs once, from the constructor, before anything else is read.
305		// `#` and `!` are both operator characters, so without this the line
306		// would otherwise tokenise as a run of operators rather than being
307		// read as a directive to skip.
308		skip_shebang_line() is
309		if _input.peek() != cast int('#') then
310		return
311		fi
312
313		_cursor.start()
314
315		let shebang_location = location
316
317		next_char()
318
319		if _input.peek() != cast int('!') then
320		prev_char('#')
321		return
322		fi
323
324		let buffer = System.Text.StringBuilder("#")
325		let c: char mut
326		do
327		c = next_char()
328		if c != '\n' then
329		buffer.append(c)
330		fi
331
332		if !(!_end_of_file /\ c != '\n') then
333		break
334		fi
335		od
336
337		_trivia.add(
338		TRIVIA.SHEBANG(
339		buffer.to_string().trim_end(),
340		shebang_location
341		)
342		)
343		si
344
345		is_end_of_file: bool => _end_of_file
346
347		location: LOCATION => _cursor.location
348
349		character_location: LOCATION => _cursor.character_location
350
351		advance_cursor(c: char) is
352		if c == cast char(13) then
353		return
354		fi
355
356		_cursor.next_column()
357
358		if c == '\n' then
359		_cursor.next_line()
360		fi
361		si
362
363		next_char() -> char is
364		let c: char mut
365
366		_read_end_of_file = false
367
368		if _prev_char != cast char(0) then
369		c = _prev_char
370		advance_cursor(c)
371		_prev_char = cast char(0)
372		return c
373		fi
374
375		if _end_of_file then
376		_read_end_of_file = true
377		return ' '
378		fi
379
380		let c0: int mut = _input.read()
381
382		if c0 == 13 then
383		c0 = 32
384		fi
385
386		if c0 == -1 \/ c0 == 12 then
387		_end_of_file = true
388		_read_end_of_file = true
389		return ' '
390		fi
391
392		c = cast char(c0)
393
394		_cursor.save()
395		advance_cursor(c)
396
397		return c
398		si
399
400		// Pushing back the end of the input leaves everything as it is: it
401		// moved the cursor nowhere, and reading again gives it again.
402		// Restoring would move the cursor back over the character before
403		// it, cutting the input's last token short by one.
404		prev_char(c: char) is
405		if _read_end_of_file then
406		return
407		fi
408
409		_cursor.restore()
410		_prev_char = c
411		si
412
413		current_string: string =>
414		_token_pair.value_string
415
416		read_escape() -> char is
417		let c: char mut = next_char()
418		let result: int mut = 0
419		if c == 't' then
420		return cast char(9)
421		elif c == 'n' then
422		return '\n'
423		elif c == 'r' then
424		return cast char(13)
425		elif c == '\\' then
426		return '\\'
427		elif c == 'u' then
428		// Exactly four hex digits, which is exactly one UTF-16 code
429		// unit and so exactly one char. Fixed width so that the
430		// escape never consumes text that follows it: the characters
431		// after a code are letters as often as not, and a-f are
432		// letters.
433		let escape_location = character_location
434
435		for i in 0..4 do
436		c = next_char()
437
438		if c >= '0' /\ c <= '9' then
439		result = 16 * result + cast int(c - '0')
440		elif c >= 'a' /\ c <= 'f' then
441		result = 16 * result + cast int(c - 'a') + 10
442		elif c >= 'A' /\ c <= 'F' then
443		result = 16 * result + cast int(c - 'A') + 10
444		else
445		prev_char(c)
446		_logger.lexer_error(escape_location, "expected four hex digits after \\u")
447		return cast char(0)
448		fi
449		od
450
451		return cast char(result)
452		elif c>='0' /\ c<='7' then
453		_logger.warn(character_location, "deprecated-octal-escape", "octal escape is deprecated - write the character as \\uXXXX")
454
455		while c >= '0' /\ c <= '7' do
456		result = 8 * result + cast int(c - '0')
457		c = next_char()
458		od
459
460		prev_char(c)
461		return cast char(result)
462		else
463		return c
464		fi
465		si
466
467		skip_white_space() -> char is
468		let c: char mut = _
469		let newlines mut = 0
470		let blank_location: LOCATION? mut = null
471		do
472		c = next_char()
473
474		if c == '\n' then
475		newlines = newlines + 1
476		if newlines == 2 then
477		blank_location = character_location
478		fi
479		fi
480
481		let is_white_space = (c==' ' \/ c==cast char(9) \/ c=='\n')
482
483		let should_break = is_end_of_file \/ !is_white_space
484
485		if is_end_of_file \/ !is_white_space then
486		break
487		fi
488		od
489
490		if newlines >= 2 /\ blank_location? then
491		_trivia.add(TRIVIA.BLANK_LINE(blank_location))
492		fi
493
494		return c
495		si
496
497		expect_format_specifier() is
498		_expect_format_string = true
499		si
500
501		read_token() -> TOKEN_PAIR is
502		let r: TOKEN mut
503
504		let c mut = skip_white_space()
505
506		if _end_of_file then
507		return TOKEN_PAIR(TOKEN.END_OF_INPUT, location, "")
508		fi
509
510		_cursor.start()
511
512		let _buffer mut = System.Text.StringBuilder()
513
514		if _expect_format_string then
515		_expect_format_string = false
516
517		_buffer = System.Text.StringBuilder()
518
519		while !_end_of_file /\ c != '\n' /\ c != '}' /\ c != '"' do
520		_buffer.append(c)
521
522		c = next_char()
523		od
524
525		if c == '}' then
526		prev_char(c)
527		elif c == '\n' then
528		prev_char(c)
529		_logger.lexer_error(character_location, "newline in interpolation format string")
530		return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
531		elif c == '"' then
532		_logger.lexer_error(character_location, "expected '}}' after format string")
533		return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
534		elif _end_of_file then
535		_logger.lexer_error(character_location, "end of file in interpolation format string")
536		end_of_input_error_locations.add(character_location)
537		return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string())
538		fi
539
540		return TOKEN_PAIR(TOKEN.FORMAT_STRING, location, _buffer.to_string())
541
542		elif c >= '0' /\ c <= '9' then
543		let is_float mut = false
544
545		_buffer = System.Text.StringBuilder()
546		_buffer.append(c)
547		c = next_char()
548
549		if c=='x' \/ c=='X' then
550		_buffer.append(c)
551		c = next_char()
552		while
553		(c>='0'/\c<='9') \/ (c>='A'/\c<='F') \/ (c>='a'/\c<='f') \/ c == '_'
554		do
555		if c != '_' then
556		_buffer.append(c)
557		fi
558		c = next_char()
559		od
560		else
561		let seen_dot mut = false
562		let pc: char mut = _
563
564		while
565		(c >= '0' /\ c <= '9') \/ c == '.' \/ c == '_'
566		do
567		if c == '.' then
568		if pc == '.' then
569		is_float = false
570
571		// bodge: can only push back one char, so use Unicode '‥' to signal we actually want to push back '..'
572		c = '‥'
573		break
574		elif seen_dot then
575		break
576		else
577		is_float = true
578		seen_dot = true
579		fi
580		else
581		if pc == '.' then
582		_buffer.append('.')
583		fi
584
585		if c != '_' then
586		_buffer.append(c)
587		fi
588		fi
589
590		pc = c
591		c = next_char()
592		od
593		fi
594
595		if is_float then
596		if c == 'e' \/ c == 'E' then
597		_buffer.append(c)
598		c = next_char()
599
600		if c == '-' then
601		_buffer.append(c)
602		c = next_char()
603		fi
604
605		if c >= '0' \/ c <= '9' \/ c == '_' then
606		while (c >= '0' /\ c <= '9') \/ c == '_' do
607		if c != '_' then
608		_buffer.append(c)
609		fi
610
611		c = next_char()
612		od
613		else
614		_logger.lexer_error(location, "expected exponent in float literal")
615		fi
616		fi
617
618		if c == 's' \/ c == 'S' \/ c == 'd' \/ c == 'D' \/ c == 'm' \/ c == 'M' then
619		_buffer.append(c)
620		c = next_char()
621		fi
622		else
623		if c == 'm' \/ c == 'M' then
624		_buffer.append(c)
625		c = next_char()
626		is_float = true
627		elif c == '`' then
628		// A backtick attaches a type suffix that would
629		// otherwise be read as a digit of the literal's own
630		// radix - `b` and `c` are hex digits as well as size
631		// selectors, so in hex they are only ever reachable
632		// this way. The backtick stays in the token text: it
633		// is what tells the classifier the suffix is explicit.
634		_buffer.append(c)
635		c = next_char()
636
637		let suffix_start = _buffer.length
638
639		if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then
640		_buffer.append(c)
641		c = next_char()
642		fi
643
644		if "bBcCsSiIlLnNwW".contains(c) then
645		_buffer.append(c)
646		c = next_char()
647		fi
648
649		if _buffer.length == suffix_start then
650		_logger.lexer_error(character_location, "expected a type suffix after ` in numeric literal")
651		fi
652		else
653		if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then
654		_buffer.append(c)
655		c = next_char()
656		fi
657
658		if "bBcCsSiIlLnNwW".contains(c) then
659		_buffer.append(c)
660		c = next_char()
661		fi
662		fi
663		fi
664
665		prev_char(c)
666
667		if is_float then
668		return TOKEN_PAIR(TOKEN.FLOAT_LITERAL, location, _buffer.to_string())
669		else
670		return TOKEN_PAIR(TOKEN.INT_LITERAL, location, _buffer.to_string())
671		fi
672
673		elif is_identifier_start(c) \/ c == '`' then
674		let was_escaped: bool mut = false
675
676		if c == '`' then
677		c = next_char()
678
679		if c == '[' then
680		return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "")
681		elif is_operator_char(c) then
682		let o = read_operator(c)
683		return TOKEN_PAIR(TOKEN.IDENTIFIER, o.location, o.value_string)
684		else
685		was_escaped = true
686		fi
687		fi
688
689		_buffer = System.Text.StringBuilder()
690
691		while is_identifier_part(c) \/ is_format_char(c) do
692		if is_format_char(c) then
693		_logger.lexer_error(
694		character_location,
695		"format character U+{cast int(c):X4} in identifier")
696		else
697		_buffer.append(c)
698		fi
699
700		c = next_char()
701		od
702
703		prev_char(c)
704		let s = _buffer.to_string()
705
706		if was_escaped then
707		if s.length == 0 then
708		_logger.lexer_error(location, "expected an identifier after backtick")
709		fi
710
711		r = TOKEN.IDENTIFIER
712		else
713		r = _symbol_tokens[s]
714		fi
715
716		return TOKEN_PAIR(r, location, s)
717		elif c == '/' then
718		let comment_location = location
719		c = next_char()
720		if c == '/' then
721		read_line_comment(comment_location)
722
723		return read_token()
724		elif c == '*' then
725		read_block_comment(comment_location)
726
727		return read_token()
728		else
729		prev_char(c)
730		return read_operator('/')
731		fi
732		elif is_operator_char(c) then
733		return read_operator(c)
734		fi
735
736		case c
737
738		when '\'' then
739		_buffer = System.Text.StringBuilder()
740
741		c = next_char()
742		while c != cast char(39) do
743		if c == cast char(92) then
744		c = read_escape()
745		_buffer.append(c)
746		c = next_char()
747		else
748		_buffer.append(c)
749		c = next_char()
750		fi
751
752		if _end_of_file then
753		_logger.lexer_error(location, "end of file in character literal")
754		end_of_input_error_locations.add(location)
755		break
756		fi
757		od
758
759		if _buffer.length < 1 then
760		_logger.lexer_error(location, "zero length character literal")
761		elif _buffer.length > 1 then
762		_logger.lexer_error(location, "character literal is too long")
763		fi
764		return TOKEN_PAIR(TOKEN.CHAR_LITERAL, location, _buffer.to_string())
765
766		when '(' then
767		return TOKEN_PAIR(TOKEN.PAREN_OPEN, location, "")
768
769		when ')' then
770		return TOKEN_PAIR(TOKEN.PAREN_CLOSE, location, "")
771
772		when '[' then
773		c = next_char()
774		if _end_of_file then
775		return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "")
776		elif c == ']' then
777		return TOKEN_PAIR(TOKEN.ARRAY_DEF, location, "")
778		else
779		prev_char(c)
780		return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "")
781		fi
782
783		when ']' then
784		return TOKEN_PAIR(TOKEN.SQUARE_CLOSE, location, "")
785
786		when ',' then
787		return TOKEN_PAIR(TOKEN.COMMA, location, "")
788
789		when ';' then
790		return TOKEN_PAIR(TOKEN.SEMICOLON, location, "")
791
792		when '‥' then
793		return read_operator(System.Text.StringBuilder(".."), next_char())
794
795		when '⟦' then
796		return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "")
797
798		when '"' then
799		return string_enter()
800
801		when '}' then
802		return interpolation_exit('}')
803
804		else
805		return TOKEN_PAIR(TOKEN.UNKNOWN, location, "{c}")
806		esac
807		si
808
809		read_operator(c: char mut) -> TOKEN_PAIR =>
810		read_operator(System.Text.StringBuilder(), c)
811
812		// `seed` carries the characters the caller has already consumed on
813		// this token's behalf. The numeric-literal scanner can push back only
814		// one character, so it hands a `..` following a number over as the
815		// single `‥`; seeding with the two dots lets the scan continue into
816		// whatever operator characters follow, so `1..<2` and `i..<2` produce
817		// the same token.
818		read_operator(seed: System.Text.StringBuilder, c: char mut) -> TOKEN_PAIR is
819		let _buffer = seed
820		let first = if _buffer.length > 0 then _buffer[0] else c fi
821
822		// A `.` immediately after a bare `!` or `?` ends the operator: it
823		// begins a member access (`x!.foo`, `x?.foo`), not a longer infix
824		// operator. Longer operators starting with `!`/`?` (e.g. `!=`) and
825		// dot operators (`..`) are unaffected.
826		while
827		is_operator_char(c) /\
828		!(_buffer.length == 1 /\ (first == '!' \/ first == '?') /\ c == '.')
829		do
830		_buffer.append(c)
831		c = next_char()
832		od
833
834		prev_char(c)
835
836		if _buffer.length < 1 then
837		return TOKEN_PAIR(TOKEN.FIRST, location, "")
838		fi
839
840		let s = _buffer.to_string()
841
842		let r mut = TOKEN.OPERATOR
843
844		if _operator_tokens.contains_key(s) then
845		r = _operator_tokens[s]
846		fi
847
848		return TOKEN_PAIR(r, location, s)
849		si
850
851		read_string_fragment() -> (fragment: string, c: char) is
852		let fragment = System.Text.StringBuilder()
853
854		let c mut = next_char()
855
856		while !_end_of_file /\ c != '\n' /\ c != '"' do
857		if c == cast char(92) then
858		c = read_escape()
859		fragment.append(c)
860		c = next_char()
861		elif c == '{' then
862		c = next_char()
863
864		if c == '{' then
865		fragment.append(c)
866		c = next_char()
867		else
868		prev_char(c)
869		c = '{'
870
871		break
872		fi
873		elif c == '}' then
874		c = next_char()
875
876		if c == '}' then
877		fragment.append(c)
878		c = next_char()
879		else
880		_logger.lexer_error(character_location, "unmatched '}}' in string interpolation")
881
882		fragment.append('}')
883		fi
884		else
885		fragment.append(c)
886		c = next_char()
887		fi
888		od
889
890		if c == '\n' then
891		_logger.lexer_error(character_location, "newline in string literal")
892		elif _end_of_file then
893		_logger.lexer_error(character_location, "end of file in string literal")
894		end_of_input_error_locations.add(character_location)
895		fi
896
897		return (fragment.to_string(), c)
898		si
899
900		// Consume the rest of a line comment whose `//` has been read,
901		// recording it as trivia.
902		read_line_comment(comment_location: LOCATION) is
903		let buffer = System.Text.StringBuilder("//")
904		let c: char mut
905		do
906		c = next_char()
907		if c != '\n' then
908		buffer.append(c)
909		fi
910
911		if !(!_end_of_file/\c!='\n') then
912		break
913		fi
914		od
915
916		_trivia.add(
917		TRIVIA.LINE_COMMENT(
918		buffer.to_string().trim_end(),
919		comment_location
920		)
921		)
922		si
923
924		// Consume the rest of a block comment whose `/*` has been read,
925		// recording it as trivia.
926		read_block_comment(comment_location: LOCATION) is
927		let buffer = System.Text.StringBuilder("/*")
928		let c: char mut
929		do
930		c = next_char()
931
932		if c == '*' then
933		c = next_char()
934		if c == '/' then
935		break
936		fi
937
938		buffer.append('*')
939		fi
940
941		buffer.append(c)
942
943		if _end_of_file then
944		break
945		fi
946		od
947
948		buffer.append("*/")
949
950		_trivia.add(
951		TRIVIA.BLOCK_COMMENT(
952		buffer.to_string(),
953		comment_location
954		)
955		)
956		si
957
958		// Adjacent string-literal concatenation: while a closed `"..."`
959		// fragment is followed (after whitespace or comments) by another
960		// `"`, fold the next fragment into this token. Stops at the first
961		// fragment that doesn't close with `"` (i.e. starts an
962		// interpolation or hits a newline). Comments between fragments are
963		// recorded as trivia and chained across, so stripping them cannot
964		// change what the token means; only a `;` separates two
965		// otherwise-adjacent literals.
966		//
967		// We treat the whole `"..." "..." "..."` run as a single token.
968		// The token's end position is wherever the final fragment
969		// terminated — we snapshot the cursor's location after each
970		// successful fragment close. When the chain fails (we consume
971		// whitespace then see a non-`"` char), the cursor is past the
972		// whitespace, but we return the snapshot taken at the previous
973		// close so the token's location end isn't smeared across the
974		// following whitespace. The stream stays at the non-`"` char;
975		// the next read_token picks up from there normally.
976		chain_adjacent_string_fragments(initial_fragment: string, initial_terminator: char) -> (fragment: string, c: char, loc: Source.LOCATION) is
977		let buffer = System.Text.StringBuilder()
978		buffer.append(initial_fragment)
979		let c mut = initial_terminator
980		let saved_loc mut = location
981
982		let done mut = false
983		while !done /\ c == '"' do
984		let peeked = skip_white_space()
985		if peeked == '"' then
986		let (next_fragment, next_terminator) = read_string_fragment()
987		buffer.append(next_fragment)
988		c = next_terminator
989		saved_loc = location
990		elif
991		peeked == '/' /\
992		(_input.peek() == cast int('/') \/ _input.peek() == cast int('*'))
993		then
994		// A comment counts as the whitespace it sits in, here as
995		// everywhere. The raw peek is sound: the pushback slot is
996		// always empty after skip_white_space hands a char out, so
997		// the reader's next character is the one after the `/`.
998		let comment_location = location
999
1000		if next_char() == '/' then
1001		read_line_comment(comment_location)
1002		else
1003		read_block_comment(comment_location)
1004		fi
1005		else
1006		prev_char(peeked)
1007		done = true
1008		fi
1009		od
1010
1011		return (buffer.to_string(), c, saved_loc)
1012		si
1013
1014		string_enter() -> TOKEN_PAIR is
1015		let (fragment, c) mut = read_string_fragment()
1016
1017		let loc: Source.LOCATION mut
1018		(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c)
1019
1020		// The closing quote settles this before anything else is
1021		// consulted: the adjacency lookahead peeks past a literal that
1022		// already closed, and can reach the end of the file doing it,
1023		// so the end-of-file flag says nothing about this literal once
1024		// its quote has been seen.
1025		if c == '"' then
1026		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1027		elif c == '\n' \/ _end_of_file then
1028		_interpolation_depth = 0
1029
1030		return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment)
1031		elif c == '{' then
1032		_interpolation_depth = _interpolation_depth + 1
1033
1034		return TOKEN_PAIR(TOKEN.ENTER_STRING, loc, fragment)
1035		else
1036		_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation")
1037
1038		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1039		fi
1040		si
1041
1042		interpolation_clear() is
1043		_interpolation_depth = 0
1044		si
1045
1046		interpolation_exit(c: char mut) -> TOKEN_PAIR is
1047		if _interpolation_depth == 0 then
1048		_logger.lexer_error(character_location, "unmatched '\"'")
1049		next_char()
1050		return read_token()
1051		fi
1052
1053		let fragment mut = ""
1054
1055		if c == '}' then
1056		(fragment, c) = read_string_fragment()
1057		fi
1058
1059		let loc: Source.LOCATION mut
1060		(fragment, c, loc) = chain_adjacent_string_fragments(fragment, c)
1061
1062		// Closing quote first, for the same reason as in string_enter.
1063		if c == '"' then
1064		_interpolation_depth = _interpolation_depth - 1
1065
1066		return TOKEN_PAIR(TOKEN.EXIT_STRING, loc, fragment)
1067		elif c == '\n' \/ _end_of_file then
1068		_interpolation_depth = 0
1069		return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment)
1070		elif c == '{' then
1071		return TOKEN_PAIR(TOKEN.CONTINUE_STRING, loc, fragment)
1072		else
1073		_logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation")
1074
1075		return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment)
1076		fi
1077		si
1078		si
1079		si