Appearance
| 1 | namespace Lexical is | |
| 2 | use System.Exception | |
| 3 | ||
| 4 | use Logging | |
| 5 | use Source | |
| 6 | ||
| 7 | ||
| 8 | use Ghul.Pipes | |
| 9 | ||
| 10 | get_stack_trace() -> string is | |
| 11 | try | |
| 12 | return Logging.CALL_STACK.text | |
| 13 | .split(['\n']) |> | |
| 14 | skip(2) |> | |
| 15 | take(5) |> | |
| 16 | map(s => s.trim().replace("at ", "")) |> join(" ") | |
| 17 | catch e: System.Exception | |
| 18 | return "unknown" | |
| 19 | yrt | |
| 20 | si | |
| 21 | ||
| 22 | @suppress("non-pascal-case-name") | |
| 23 | class TOKENIZER_EXCEPTION(s: string): Exception is | |
| 24 | super(s) | |
| 25 | si | |
| 26 | ||
| 27 | class TOKEN_MAP is | |
| 28 | _map: Collections.MAP[string,TOKEN] | |
| 29 | ||
| 30 | init() is | |
| 31 | super.init() | |
| 32 | _map = Collections.MAP[string,TOKEN](223) | |
| 33 | si | |
| 34 | ||
| 35 | [s: string]: TOKEN public => | |
| 36 | if _map.contains_key(s) then | |
| 37 | _map[s] | |
| 38 | else | |
| 39 | TOKEN.IDENTIFIER | |
| 40 | fi, | |
| 41 | = t is | |
| 42 | _map[s] = t | |
| 43 | si | |
| 44 | si | |
| 45 | ||
| 46 | class TOKEN_PAIR(token: TOKEN, location: LOCATION, value: string init) is | |
| 47 | value_string: string | |
| 48 | ||
| 49 | // True when this token is the first on its source line. Stamped by | |
| 50 | // TOKEN_LOOKAHEAD as tokens are first read, so it survives | |
| 51 | // speculation replays. | |
| 52 | first_on_line: bool public | |
| 53 | ||
| 54 | // True when the token before this one closed a string literal. | |
| 55 | // Stamped alongside first_on_line. A `;` carrying this, followed | |
| 56 | // by another string, is separating fragments that would otherwise | |
| 57 | // chain into one literal, so it is never redundant. | |
| 58 | follows_string: bool public | |
| 59 | ||
| 60 | name: string => TOKEN_NAMES[token] | |
| 61 | ||
| 62 | to_string() -> string => | |
| 63 | "{location}: {TOKEN_NAMES[token]} {if value_string.length > 0 then "'{value_string}'" else "" fi}" | |
| 64 | ||
| 65 | to_short_string() -> string => | |
| 66 | "{if value_string.length > 0 /\ value_string !~ TOKEN_NAMES[token] then "{TOKEN_NAMES[token]} \"{value_string}\"" else "'{TOKEN_NAMES[token]}'" fi}" | |
| 67 | ||
| 68 | init(..) is | |
| 69 | value_string = value | |
| 70 | si | |
| 71 | si | |
| 72 | ||
| 73 | class TOKENIZER: TokenSource is | |
| 74 | _prev_count: int | |
| 75 | ||
| 76 | // Where a literal the input ended inside was reported: a string, | |
| 77 | // a character literal or an interpolation format with no closing | |
| 78 | // quote before end of input, which more input would close. | |
| 79 | end_of_input_error_locations: Collections.LIST[LOCATION] | |
| 80 | ||
| 81 | _logger: Logger | |
| 82 | _token_pair: TOKEN_PAIR | |
| 83 | _symbol_tokens: TOKEN_MAP static | |
| 84 | _operator_chars: Collections.SET[char] static | |
| 85 | _operator_tokens: Collections.MAP[string,TOKEN] static | |
| 86 | _input: IO.TextReader | |
| 87 | _end_of_file: bool | |
| 88 | _prev_char: char | |
| 89 | ||
| 90 | // Whether the last character read was the space that stands for the | |
| 91 | // end of the input, which moved the cursor nowhere. | |
| 92 | _read_end_of_file: bool | |
| 93 | _cursor: LOCATION_CURSOR | |
| 94 | ||
| 95 | _interpolation_depth: int | |
| 96 | ||
| 97 | // How many braces are open in the code at each level of string | |
| 98 | // interpolation, outermost first, so a `}` closes a brace the code | |
| 99 | // opened before it ends an interpolation. Always one entry more | |
| 100 | // than the interpolation depth. | |
| 101 | _open_braces: Collections.LIST[int] | |
| 102 | _expect_format_string: bool | |
| 103 | ||
| 104 | _trivia: Collections.LIST[TRIVIA] | |
| 105 | ||
| 106 | // The line the most recent token or comment started on, and whether | |
| 107 | // the item being read is the first thing on its line. A `///` comment | |
| 108 | // is a doc comment only when nothing precedes it on its line. | |
| 109 | _content_line: int | |
| 110 | _item_starts_line: bool | |
| 111 | ||
| 112 | // Comments and blank-line markers, in source order. Discarded by the | |
| 113 | // parser; consumed by the formatter to round-trip non-token content. | |
| 114 | trivia: Collections.Iterable[TRIVIA] => _trivia | |
| 115 | ||
| 116 | init() static is | |
| 117 | _operator_chars = Collections.SET[char]([ | |
| 118 | '!', '$', '%', '^', '&', '*', '-', '+', '=', '|', ':', '@', '~', '#', '\\', '<', '>', '.', '?', '/' | |
| 119 | ]: char ) | |
| 120 | ||
| 121 | _operator_tokens = Collections.MAP[string,TOKEN]() | |
| 122 | _operator_tokens["="] = TOKEN.ASSIGN | |
| 123 | _operator_tokens[":"] = TOKEN.COLON | |
| 124 | _operator_tokens["."] = TOKEN.DOT | |
| 125 | _operator_tokens["->"] = TOKEN.ARROW_THIN | |
| 126 | _operator_tokens["=>"] = TOKEN.ARROW_FAT | |
| 127 | _operator_tokens["|>"] = TOKEN.BAR_ARROW | |
| 128 | _operator_tokens["~>"] = TOKEN.TILDE_ARROW | |
| 129 | _operator_tokens["?"] = TOKEN.QUESTION | |
| 130 | _operator_tokens["@"] = TOKEN.AT | |
| 131 | _operator_tokens["@@"] = TOKEN.AT_AT | |
| 132 | _symbol_tokens = TOKEN_MAP() | |
| 133 | _symbol_tokens["abstract"] = TOKEN.ABSTRACT | |
| 134 | _symbol_tokens["namespace"] = TOKEN.NAMESPACE | |
| 135 | _symbol_tokens["class"] = TOKEN.CLASS | |
| 136 | _symbol_tokens["struct"] = TOKEN.STRUCT | |
| 137 | _symbol_tokens["union"] = TOKEN.UNION | |
| 138 | _symbol_tokens["partial"] = TOKEN.PARTIAL | |
| 139 | _symbol_tokens["impl"] = TOKEN.IMPL | |
| 140 | _symbol_tokens["enum"] = TOKEN.ENUM | |
| 141 | _symbol_tokens["public"] = TOKEN.PUBLIC | |
| 142 | _symbol_tokens["protected"] = TOKEN.PROTECTED | |
| 143 | _symbol_tokens["private"] = TOKEN.PRIVATE | |
| 144 | _symbol_tokens["field"] = TOKEN.FIELD | |
| 145 | _symbol_tokens["static"] = TOKEN.STATIC | |
| 146 | _symbol_tokens["innate"] = TOKEN.INNATE | |
| 147 | _symbol_tokens["rec"] = TOKEN.REC | |
| 148 | _symbol_tokens["if"] = TOKEN.IF | |
| 149 | _symbol_tokens["else"] = TOKEN.ELSE | |
| 150 | _symbol_tokens["while"] = TOKEN.WHILE | |
| 151 | _symbol_tokens["do"] = TOKEN.DO | |
| 152 | _symbol_tokens["for"] = TOKEN.FOR | |
| 153 | _symbol_tokens["in"] = TOKEN.IN | |
| 154 | _symbol_tokens["case"] = TOKEN.CASE | |
| 155 | _symbol_tokens["when"] = TOKEN.WHEN | |
| 156 | _symbol_tokens["default"] = TOKEN.DEFAULT | |
| 157 | _symbol_tokens["break"] = TOKEN.BREAK | |
| 158 | _symbol_tokens["continue"] = TOKEN.CONTINUE | |
| 159 | _symbol_tokens["ref"] = TOKEN.REF | |
| 160 | _symbol_tokens["ptr"] = TOKEN.PTR | |
| 161 | _symbol_tokens["new"] = TOKEN.NEW | |
| 162 | _symbol_tokens["throw"] = TOKEN.THROW | |
| 163 | _symbol_tokens["return"] = TOKEN.RETURN | |
| 164 | _symbol_tokens["cast"] = TOKEN.CAST | |
| 165 | _symbol_tokens["try"] = TOKEN.TRY | |
| 166 | _symbol_tokens["let"] = TOKEN.LET | |
| 167 | _symbol_tokens["mut"] = TOKEN.MUT | |
| 168 | _symbol_tokens["await"] = TOKEN.AWAIT | |
| 169 | _symbol_tokens["catch"] = TOKEN.CATCH | |
| 170 | _symbol_tokens["finally"] = TOKEN.FINALLY | |
| 171 | _symbol_tokens["self"] = TOKEN.SELF | |
| 172 | _symbol_tokens["super"] = TOKEN.SUPER | |
| 173 | _symbol_tokens["null"] = TOKEN.NULL | |
| 174 | _symbol_tokens["use"] = TOKEN.USE | |
| 175 | _symbol_tokens["trait"] = TOKEN.TRAIT | |
| 176 | _symbol_tokens["isa"] = TOKEN.ISA | |
| 177 | _symbol_tokens["typeof"] = TOKEN.TYPEOF | |
| 178 | // _symbol_tokens["operator"] = TOKEN.OPERATOR; | |
| 179 | _symbol_tokens["is"] = TOKEN.IS | |
| 180 | _symbol_tokens["si"] = TOKEN.SI | |
| 181 | _symbol_tokens["then"] = TOKEN.THEN | |
| 182 | _symbol_tokens["elif"] = TOKEN.ELIF | |
| 183 | _symbol_tokens["fi"] = TOKEN.FI | |
| 184 | _symbol_tokens["esac"] = TOKEN.ESAC | |
| 185 | _symbol_tokens["lav"] = TOKEN.LAV | |
| 186 | _symbol_tokens["val"] = TOKEN.VAL | |
| 187 | _symbol_tokens["od"] = TOKEN.OD | |
| 188 | _symbol_tokens["yield"] = TOKEN.YIELD | |
| 189 | _symbol_tokens["yrt"] = TOKEN.YRT | |
| 190 | _symbol_tokens["true"] = TOKEN.TRUE | |
| 191 | _symbol_tokens["false"] = TOKEN.FALSE | |
| 192 | _symbol_tokens["assert"] = TOKEN.ASSERT | |
| 193 | si | |
| 194 | ||
| 195 | // True when a bare occurrence of `name` would tokenise as a keyword | |
| 196 | // rather than an identifier, so writing it back out unadorned would | |
| 197 | // change its meaning. Such a name can only have reached the parser as | |
| 198 | // an identifier via a backtick escape. | |
| 199 | is_reserved_word(name: string?) -> bool static is | |
| 200 | return name? /\ _symbol_tokens[name] != TOKEN.IDENTIFIER | |
| 201 | si | |
| 202 | ||
| 203 | // True when a bare occurrence of `name` would tokenise as a numeric | |
| 204 | // literal rather than an identifier: it is composed only of identifier | |
| 205 | // characters but begins with a digit. Such a name reached the parser | |
| 206 | // as an identifier via a backtick escape. | |
| 207 | is_numeric_identifier(name: string) -> bool static is | |
| 208 | if name.length == 0 then | |
| 209 | return false | |
| 210 | fi | |
| 211 | ||
| 212 | let first = name[0] | |
| 213 | ||
| 214 | if !(first >= '0' /\ first <= '9') then | |
| 215 | return false | |
| 216 | fi | |
| 217 | ||
| 218 | for c in name do | |
| 219 | if !is_identifier_part(c) then | |
| 220 | return false | |
| 221 | fi | |
| 222 | od | |
| 223 | ||
| 224 | return true | |
| 225 | si | |
| 226 | ||
| 227 | // True when `name` is composed entirely of operator characters, so a | |
| 228 | // bare occurrence would tokenise as an operator and a backtick escape | |
| 229 | // is needed to refer to it as a plain identifier. | |
| 230 | is_operator_name(name: string) -> bool static is | |
| 231 | if name.length == 0 then | |
| 232 | return false | |
| 233 | fi | |
| 234 | ||
| 235 | for c in name do | |
| 236 | if !is_operator_char(c) then | |
| 237 | return false | |
| 238 | fi | |
| 239 | od | |
| 240 | ||
| 241 | return true | |
| 242 | si | |
| 243 | ||
| 244 | // An operator character is one of the ASCII set above, or any | |
| 245 | // non-ASCII character Unicode classes as a symbol. A letter is | |
| 246 | // never a symbol, so no character is both an operator character | |
| 247 | // and an identifier character. | |
| 248 | is_operator_char(c: char) -> bool static => | |
| 249 | _operator_chars.contains(c) \/ | |
| 250 | (cast int(c) > 0x7E /\ char.is_symbol(c)) | |
| 251 | ||
| 252 | // Identifier characters follow the categories C# draws on, less | |
| 253 | // the format characters: those render as nothing, so admitting | |
| 254 | // them would let two names that read alike be different names. | |
| 255 | // Only the basic plane is covered - a character above it arrives | |
| 256 | // as a surrogate pair, which this scanner reads as two characters. | |
| 257 | is_identifier_start(c: char) -> bool static is | |
| 258 | if (c >= 'a' /\ c <= 'z') \/ (c >= 'A' /\ c <= 'Z') \/ c == '_' then | |
| 259 | return true | |
| 260 | fi | |
| 261 | ||
| 262 | if cast int(c) <= 0x7E then | |
| 263 | return false | |
| 264 | fi | |
| 265 | ||
| 266 | return char.is_letter(c) \/ _is_letter_number(c) | |
| 267 | si | |
| 268 | ||
| 269 | is_identifier_part(c: char) -> bool static is | |
| 270 | if (c >= '0' /\ c <= '9') \/ is_identifier_start(c) then | |
| 271 | return true | |
| 272 | fi | |
| 273 | ||
| 274 | if cast int(c) <= 0x7E then | |
| 275 | return false | |
| 276 | fi | |
| 277 | ||
| 278 | return _is_other_identifier_part(c) | |
| 279 | si | |
| 280 | ||
| 281 | is_format_char(c: char) -> bool static => | |
| 282 | cast int(c) > 0x7E /\ _is_format(c) | |
| 283 | ||
| 284 | @IF.dotnet() | |
| 285 | _is_letter_number(c: char) -> bool static => | |
| 286 | char.get_unicode_category(c) == System.Globalization.UnicodeCategory.LETTER_NUMBER | |
| 287 | ||
| 288 | @IF.dotnet() | |
| 289 | _is_other_identifier_part(c: char) -> bool static is | |
| 290 | let category = char.get_unicode_category(c) | |
| 291 | ||
| 292 | return category == System.Globalization.UnicodeCategory.DECIMAL_DIGIT_NUMBER \/ | |
| 293 | category == System.Globalization.UnicodeCategory.NON_SPACING_MARK \/ | |
| 294 | category == System.Globalization.UnicodeCategory.SPACING_COMBINING_MARK \/ | |
| 295 | category == System.Globalization.UnicodeCategory.CONNECTOR_PUNCTUATION | |
| 296 | si | |
| 297 | ||
| 298 | @IF.dotnet() | |
| 299 | _is_format(c: char) -> bool static => | |
| 300 | char.get_unicode_category(c) == System.Globalization.UnicodeCategory.FORMAT | |
| 301 | ||
| 302 | // Where the core library classifies only the scripts it covers, no | |
| 303 | // other character is a letter number, a combining mark or a | |
| 304 | // connector. | |
| 305 | @IF.not.dotnet() | |
| 306 | _is_letter_number(c: char) -> bool static => false | |
| 307 | ||
| 308 | @IF.not.dotnet() | |
| 309 | _is_other_identifier_part(c: char) -> bool static => false | |
| 310 | ||
| 311 | // The format characters of the basic plane, each named by code so | |
| 312 | // that none can slip into an identifier unseen. | |
| 313 | @IF.not.dotnet() | |
| 314 | _is_format(c: char) -> bool static is | |
| 315 | let code = cast int(c) | |
| 316 | ||
| 317 | return | |
| 318 | code == 0xAD \/ code >= 0x600 /\ code <= 0x605 \/ code == 0x61C \/ code == 0x6DD \/ | |
| 319 | code == 0x70F \/ code == 0x180E \/ code >= 0x200B /\ code <= 0x200F \/ | |
| 320 | code >= 0x202A /\ code <= 0x202E \/ code >= 0x2060 /\ code <= 0x2064 \/ | |
| 321 | code >= 0x2066 /\ code <= 0x206F \/ code == 0xFEFF \/ code >= 0xFFF9 /\ code <= 0xFFFB | |
| 322 | si | |
| 323 | ||
| 324 | init(logger: Logger, file_name: string, i: IO.TextReader, is_internal_file: bool) is | |
| 325 | super.init() | |
| 326 | ||
| 327 | _logger = logger | |
| 328 | ||
| 329 | _end_of_file = false | |
| 330 | _open_braces = Collections.LIST[int]([0]) | |
| 331 | end_of_input_error_locations = Collections.LIST[LOCATION]() | |
| 332 | _input = i | |
| 333 | ||
| 334 | _trivia = Collections.LIST[TRIVIA]() | |
| 335 | ||
| 336 | if is_internal_file then | |
| 337 | _cursor = INTERNAL_LOCATION_CURSOR() | |
| 338 | else | |
| 339 | _cursor = LOCATION_CURSOR(file_name) | |
| 340 | fi | |
| 341 | ||
| 342 | _token_pair = TOKEN_PAIR(TOKEN.UNKNOWN, location, "") | |
| 343 | ||
| 344 | skip_shebang_line() | |
| 345 | si | |
| 346 | ||
| 347 | // A `#!...` line is only meaningful as the file's first two bytes - | |
| 348 | // the kernel's own rule for an executable script - so this only ever | |
| 349 | // runs once, from the constructor, before anything else is read. | |
| 350 | // `#` and `!` are both operator characters, so without this the line | |
| 351 | // would otherwise tokenise as a run of operators rather than being | |
| 352 | // read as a directive to skip. | |
| 353 | skip_shebang_line() is | |
| 354 | if _input.peek() != cast int('#') then | |
| 355 | return | |
| 356 | fi | |
| 357 | ||
| 358 | _cursor.start() | |
| 359 | ||
| 360 | let shebang_location = location | |
| 361 | ||
| 362 | next_char() | |
| 363 | ||
| 364 | if _input.peek() != cast int('!') then | |
| 365 | prev_char('#') | |
| 366 | return | |
| 367 | fi | |
| 368 | ||
| 369 | let buffer = System.Text.StringBuilder("#") | |
| 370 | let c: char mut | |
| 371 | do | |
| 372 | c = next_char() | |
| 373 | if c != '\n' then | |
| 374 | buffer.append(c) | |
| 375 | fi | |
| 376 | ||
| 377 | if !(!_end_of_file /\ c != '\n') then | |
| 378 | break | |
| 379 | fi | |
| 380 | od | |
| 381 | ||
| 382 | _trivia.add( | |
| 383 | TRIVIA.SHEBANG( | |
| 384 | buffer.to_string().trim_end(), | |
| 385 | shebang_location | |
| 386 | ) | |
| 387 | ) | |
| 388 | si | |
| 389 | ||
| 390 | is_end_of_file: bool => _end_of_file | |
| 391 | ||
| 392 | location: LOCATION => _cursor.location | |
| 393 | ||
| 394 | character_location: LOCATION => _cursor.character_location | |
| 395 | ||
| 396 | advance_cursor(c: char) is | |
| 397 | if c == cast char(13) then | |
| 398 | return | |
| 399 | fi | |
| 400 | ||
| 401 | _cursor.next_column() | |
| 402 | ||
| 403 | if c == '\n' then | |
| 404 | _cursor.next_line() | |
| 405 | fi | |
| 406 | si | |
| 407 | ||
| 408 | next_char() -> char is | |
| 409 | let c: char mut | |
| 410 | ||
| 411 | _read_end_of_file = false | |
| 412 | ||
| 413 | if _prev_char != cast char(0) then | |
| 414 | c = _prev_char | |
| 415 | advance_cursor(c) | |
| 416 | _prev_char = cast char(0) | |
| 417 | return c | |
| 418 | fi | |
| 419 | ||
| 420 | if _end_of_file then | |
| 421 | _read_end_of_file = true | |
| 422 | return ' ' | |
| 423 | fi | |
| 424 | ||
| 425 | let c0: int mut = _input.read() | |
| 426 | ||
| 427 | if c0 == 13 then | |
| 428 | c0 = 32 | |
| 429 | fi | |
| 430 | ||
| 431 | if c0 == -1 \/ c0 == 12 then | |
| 432 | _end_of_file = true | |
| 433 | _read_end_of_file = true | |
| 434 | return ' ' | |
| 435 | fi | |
| 436 | ||
| 437 | c = cast char(c0) | |
| 438 | ||
| 439 | _cursor.save() | |
| 440 | advance_cursor(c) | |
| 441 | ||
| 442 | return c | |
| 443 | si | |
| 444 | ||
| 445 | // Pushing back the end of the input leaves everything as it is: it | |
| 446 | // moved the cursor nowhere, and reading again gives it again. | |
| 447 | // Restoring would move the cursor back over the character before | |
| 448 | // it, cutting the input's last token short by one. | |
| 449 | prev_char(c: char) is | |
| 450 | if _read_end_of_file then | |
| 451 | return | |
| 452 | fi | |
| 453 | ||
| 454 | _cursor.restore() | |
| 455 | _prev_char = c | |
| 456 | si | |
| 457 | ||
| 458 | current_string: string => | |
| 459 | _token_pair.value_string | |
| 460 | ||
| 461 | read_escape() -> char is | |
| 462 | let c: char mut = next_char() | |
| 463 | let result: int mut = 0 | |
| 464 | if c == 't' then | |
| 465 | return cast char(9) | |
| 466 | elif c == 'n' then | |
| 467 | return '\n' | |
| 468 | elif c == 'r' then | |
| 469 | return cast char(13) | |
| 470 | elif c == '\\' then | |
| 471 | return '\\' | |
| 472 | elif c == 'u' then | |
| 473 | // Exactly four hex digits, which is exactly one UTF-16 code | |
| 474 | // unit and so exactly one char. Fixed width so that the | |
| 475 | // escape never consumes text that follows it: the characters | |
| 476 | // after a code are letters as often as not, and a-f are | |
| 477 | // letters. | |
| 478 | let escape_location = character_location | |
| 479 | ||
| 480 | for i in 0..4 do | |
| 481 | c = next_char() | |
| 482 | ||
| 483 | if c >= '0' /\ c <= '9' then | |
| 484 | result = 16 * result + cast int(c - '0') | |
| 485 | elif c >= 'a' /\ c <= 'f' then | |
| 486 | result = 16 * result + cast int(c - 'a') + 10 | |
| 487 | elif c >= 'A' /\ c <= 'F' then | |
| 488 | result = 16 * result + cast int(c - 'A') + 10 | |
| 489 | else | |
| 490 | prev_char(c) | |
| 491 | _logger.lexer_error(escape_location, "expected four hex digits after \\u") | |
| 492 | return cast char(0) | |
| 493 | fi | |
| 494 | od | |
| 495 | ||
| 496 | return cast char(result) | |
| 497 | elif c>='0' /\ c<='7' then | |
| 498 | _logger.warn(character_location, "deprecated-octal-escape", "octal escape is deprecated - write the character as \\uXXXX") | |
| 499 | ||
| 500 | while c >= '0' /\ c <= '7' do | |
| 501 | result = 8 * result + cast int(c - '0') | |
| 502 | c = next_char() | |
| 503 | od | |
| 504 | ||
| 505 | prev_char(c) | |
| 506 | return cast char(result) | |
| 507 | else | |
| 508 | return c | |
| 509 | fi | |
| 510 | si | |
| 511 | ||
| 512 | skip_white_space() -> char is | |
| 513 | let c: char mut = _ | |
| 514 | let newlines mut = 0 | |
| 515 | let blank_location: LOCATION? mut = null | |
| 516 | do | |
| 517 | c = next_char() | |
| 518 | ||
| 519 | if c == '\n' then | |
| 520 | newlines = newlines + 1 | |
| 521 | if newlines == 2 then | |
| 522 | blank_location = character_location | |
| 523 | fi | |
| 524 | fi | |
| 525 | ||
| 526 | let is_white_space = (c==' ' \/ c==cast char(9) \/ c=='\n') | |
| 527 | ||
| 528 | let should_break = is_end_of_file \/ !is_white_space | |
| 529 | ||
| 530 | if is_end_of_file \/ !is_white_space then | |
| 531 | break | |
| 532 | fi | |
| 533 | od | |
| 534 | ||
| 535 | if newlines >= 2 /\ blank_location? then | |
| 536 | _trivia.add(TRIVIA.BLANK_LINE(blank_location)) | |
| 537 | fi | |
| 538 | ||
| 539 | return c | |
| 540 | si | |
| 541 | ||
| 542 | expect_format_specifier() is | |
| 543 | _expect_format_string = true | |
| 544 | si | |
| 545 | ||
| 546 | read_token() -> TOKEN_PAIR is | |
| 547 | let r: TOKEN mut | |
| 548 | ||
| 549 | let c mut = skip_white_space() | |
| 550 | ||
| 551 | if _end_of_file then | |
| 552 | return TOKEN_PAIR(TOKEN.END_OF_INPUT, location, "") | |
| 553 | fi | |
| 554 | ||
| 555 | _cursor.start() | |
| 556 | ||
| 557 | _note_item_start(location.start_line) | |
| 558 | ||
| 559 | let _buffer mut = System.Text.StringBuilder() | |
| 560 | ||
| 561 | if _expect_format_string then | |
| 562 | _expect_format_string = false | |
| 563 | ||
| 564 | _buffer = System.Text.StringBuilder() | |
| 565 | ||
| 566 | while !_end_of_file /\ c != '\n' /\ c != '}' /\ c != '"' do | |
| 567 | _buffer.append(c) | |
| 568 | ||
| 569 | c = next_char() | |
| 570 | od | |
| 571 | ||
| 572 | if c == '}' then | |
| 573 | prev_char(c) | |
| 574 | elif c == '\n' then | |
| 575 | prev_char(c) | |
| 576 | _logger.lexer_error(character_location, "newline in interpolation format string") | |
| 577 | return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string()) | |
| 578 | elif c == '"' then | |
| 579 | _logger.lexer_error(character_location, "expected '}}' after format string") | |
| 580 | return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string()) | |
| 581 | elif _end_of_file then | |
| 582 | _logger.lexer_error(character_location, "end of file in interpolation format string") | |
| 583 | end_of_input_error_locations.add(character_location) | |
| 584 | return TOKEN_PAIR(TOKEN.CANCEL_STRING, location, _buffer.to_string()) | |
| 585 | fi | |
| 586 | ||
| 587 | return TOKEN_PAIR(TOKEN.FORMAT_STRING, location, _buffer.to_string()) | |
| 588 | ||
| 589 | elif c >= '0' /\ c <= '9' then | |
| 590 | let is_float mut = false | |
| 591 | ||
| 592 | _buffer = System.Text.StringBuilder() | |
| 593 | _buffer.append(c) | |
| 594 | c = next_char() | |
| 595 | ||
| 596 | if c=='x' \/ c=='X' then | |
| 597 | _buffer.append(c) | |
| 598 | c = next_char() | |
| 599 | while | |
| 600 | (c>='0'/\c<='9') \/ (c>='A'/\c<='F') \/ (c>='a'/\c<='f') \/ c == '_' | |
| 601 | do | |
| 602 | if c != '_' then | |
| 603 | _buffer.append(c) | |
| 604 | fi | |
| 605 | c = next_char() | |
| 606 | od | |
| 607 | else | |
| 608 | let seen_dot mut = false | |
| 609 | let pc: char mut = _ | |
| 610 | ||
| 611 | while | |
| 612 | (c >= '0' /\ c <= '9') \/ c == '.' \/ c == '_' | |
| 613 | do | |
| 614 | if c == '.' then | |
| 615 | if pc == '.' then | |
| 616 | is_float = false | |
| 617 | ||
| 618 | // bodge: can only push back one char, so use Unicode '‥' to signal we actually want to push back '..' | |
| 619 | c = '‥' | |
| 620 | break | |
| 621 | elif seen_dot then | |
| 622 | break | |
| 623 | else | |
| 624 | is_float = true | |
| 625 | seen_dot = true | |
| 626 | fi | |
| 627 | else | |
| 628 | if pc == '.' then | |
| 629 | _buffer.append('.') | |
| 630 | fi | |
| 631 | ||
| 632 | if c != '_' then | |
| 633 | _buffer.append(c) | |
| 634 | fi | |
| 635 | fi | |
| 636 | ||
| 637 | pc = c | |
| 638 | c = next_char() | |
| 639 | od | |
| 640 | fi | |
| 641 | ||
| 642 | if is_float then | |
| 643 | if c == 'e' \/ c == 'E' then | |
| 644 | _buffer.append(c) | |
| 645 | c = next_char() | |
| 646 | ||
| 647 | if c == '-' then | |
| 648 | _buffer.append(c) | |
| 649 | c = next_char() | |
| 650 | fi | |
| 651 | ||
| 652 | if c >= '0' \/ c <= '9' \/ c == '_' then | |
| 653 | while (c >= '0' /\ c <= '9') \/ c == '_' do | |
| 654 | if c != '_' then | |
| 655 | _buffer.append(c) | |
| 656 | fi | |
| 657 | ||
| 658 | c = next_char() | |
| 659 | od | |
| 660 | else | |
| 661 | _logger.lexer_error(location, "expected exponent in float literal") | |
| 662 | fi | |
| 663 | fi | |
| 664 | ||
| 665 | if c == 's' \/ c == 'S' \/ c == 'd' \/ c == 'D' \/ c == 'm' \/ c == 'M' then | |
| 666 | _buffer.append(c) | |
| 667 | c = next_char() | |
| 668 | fi | |
| 669 | else | |
| 670 | if c == 'm' \/ c == 'M' then | |
| 671 | _buffer.append(c) | |
| 672 | c = next_char() | |
| 673 | is_float = true | |
| 674 | elif c == '`' then | |
| 675 | // A backtick attaches a type suffix that would | |
| 676 | // otherwise be read as a digit of the literal's own | |
| 677 | // radix - `b` and `c` are hex digits as well as size | |
| 678 | // selectors, so in hex they are only ever reachable | |
| 679 | // this way. The backtick stays in the token text: it | |
| 680 | // is what tells the classifier the suffix is explicit. | |
| 681 | _buffer.append(c) | |
| 682 | c = next_char() | |
| 683 | ||
| 684 | let suffix_start = _buffer.length | |
| 685 | ||
| 686 | if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then | |
| 687 | _buffer.append(c) | |
| 688 | c = next_char() | |
| 689 | fi | |
| 690 | ||
| 691 | if "bBcCsSiIlLnNwW".contains(c) then | |
| 692 | _buffer.append(c) | |
| 693 | c = next_char() | |
| 694 | fi | |
| 695 | ||
| 696 | if _buffer.length == suffix_start then | |
| 697 | _logger.lexer_error(character_location, "expected a type suffix after ` in numeric literal") | |
| 698 | fi | |
| 699 | else | |
| 700 | if c == 's' \/ c == 'S' \/ c == 'u' \/ c == 'U' then | |
| 701 | _buffer.append(c) | |
| 702 | c = next_char() | |
| 703 | fi | |
| 704 | ||
| 705 | if "bBcCsSiIlLnNwW".contains(c) then | |
| 706 | _buffer.append(c) | |
| 707 | c = next_char() | |
| 708 | fi | |
| 709 | fi | |
| 710 | fi | |
| 711 | ||
| 712 | prev_char(c) | |
| 713 | ||
| 714 | if is_float then | |
| 715 | return TOKEN_PAIR(TOKEN.FLOAT_LITERAL, location, _buffer.to_string()) | |
| 716 | else | |
| 717 | return TOKEN_PAIR(TOKEN.INT_LITERAL, location, _buffer.to_string()) | |
| 718 | fi | |
| 719 | ||
| 720 | elif is_identifier_start(c) \/ c == '`' then | |
| 721 | let was_escaped: bool mut = false | |
| 722 | ||
| 723 | if c == '`' then | |
| 724 | c = next_char() | |
| 725 | ||
| 726 | if c == '[' then | |
| 727 | return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "") | |
| 728 | elif is_operator_char(c) then | |
| 729 | let o = read_operator(c) | |
| 730 | return TOKEN_PAIR(TOKEN.IDENTIFIER, o.location, o.value_string) | |
| 731 | else | |
| 732 | was_escaped = true | |
| 733 | fi | |
| 734 | fi | |
| 735 | ||
| 736 | _buffer = System.Text.StringBuilder() | |
| 737 | ||
| 738 | while is_identifier_part(c) \/ is_format_char(c) do | |
| 739 | if is_format_char(c) then | |
| 740 | _logger.lexer_error( | |
| 741 | character_location, | |
| 742 | "format character U+{cast int(c):X4} in identifier") | |
| 743 | else | |
| 744 | _buffer.append(c) | |
| 745 | fi | |
| 746 | ||
| 747 | c = next_char() | |
| 748 | od | |
| 749 | ||
| 750 | prev_char(c) | |
| 751 | let s = _buffer.to_string() | |
| 752 | ||
| 753 | if was_escaped then | |
| 754 | if s.length == 0 then | |
| 755 | _logger.lexer_error(location, "expected an identifier after backtick") | |
| 756 | fi | |
| 757 | ||
| 758 | r = TOKEN.IDENTIFIER | |
| 759 | else | |
| 760 | r = _symbol_tokens[s] | |
| 761 | fi | |
| 762 | ||
| 763 | return TOKEN_PAIR(r, location, s) | |
| 764 | elif c == '/' then | |
| 765 | let comment_location = location | |
| 766 | c = next_char() | |
| 767 | if c == '/' then | |
| 768 | read_line_comment(comment_location) | |
| 769 | ||
| 770 | return read_token() | |
| 771 | elif c == '*' then | |
| 772 | read_block_comment(comment_location) | |
| 773 | ||
| 774 | return read_token() | |
| 775 | else | |
| 776 | prev_char(c) | |
| 777 | return read_operator('/') | |
| 778 | fi | |
| 779 | elif is_operator_char(c) then | |
| 780 | return read_operator(c) | |
| 781 | fi | |
| 782 | ||
| 783 | case c | |
| 784 | ||
| 785 | when '\'' then | |
| 786 | _buffer = System.Text.StringBuilder() | |
| 787 | ||
| 788 | c = next_char() | |
| 789 | while c != cast char(39) do | |
| 790 | if c == cast char(92) then | |
| 791 | c = read_escape() | |
| 792 | _buffer.append(c) | |
| 793 | c = next_char() | |
| 794 | else | |
| 795 | _buffer.append(c) | |
| 796 | c = next_char() | |
| 797 | fi | |
| 798 | ||
| 799 | if _end_of_file then | |
| 800 | _logger.lexer_error(location, "end of file in character literal") | |
| 801 | end_of_input_error_locations.add(location) | |
| 802 | break | |
| 803 | fi | |
| 804 | od | |
| 805 | ||
| 806 | if _buffer.length < 1 then | |
| 807 | _logger.lexer_error(location, "zero length character literal") | |
| 808 | elif _buffer.length > 1 then | |
| 809 | _logger.lexer_error(location, "character literal is too long") | |
| 810 | fi | |
| 811 | return TOKEN_PAIR(TOKEN.CHAR_LITERAL, location, _buffer.to_string()) | |
| 812 | ||
| 813 | when '(' then | |
| 814 | return TOKEN_PAIR(TOKEN.PAREN_OPEN, location, "") | |
| 815 | ||
| 816 | when ')' then | |
| 817 | return TOKEN_PAIR(TOKEN.PAREN_CLOSE, location, "") | |
| 818 | ||
| 819 | when '[' then | |
| 820 | c = next_char() | |
| 821 | if _end_of_file then | |
| 822 | return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "") | |
| 823 | elif c == ']' then | |
| 824 | return TOKEN_PAIR(TOKEN.ARRAY_DEF, location, "") | |
| 825 | else | |
| 826 | prev_char(c) | |
| 827 | return TOKEN_PAIR(TOKEN.SQUARE_OPEN, location, "") | |
| 828 | fi | |
| 829 | ||
| 830 | when ']' then | |
| 831 | return TOKEN_PAIR(TOKEN.SQUARE_CLOSE, location, "") | |
| 832 | ||
| 833 | when ',' then | |
| 834 | return TOKEN_PAIR(TOKEN.COMMA, location, "") | |
| 835 | ||
| 836 | when ';' then | |
| 837 | return TOKEN_PAIR(TOKEN.SEMICOLON, location, "") | |
| 838 | ||
| 839 | when '‥' then | |
| 840 | return read_operator(System.Text.StringBuilder(".."), next_char()) | |
| 841 | ||
| 842 | when '⟦' then | |
| 843 | return TOKEN_PAIR(TOKEN.SQUARE_OPEN_TICK, location, "") | |
| 844 | ||
| 845 | when '"' then | |
| 846 | return string_enter() | |
| 847 | ||
| 848 | when '{' then | |
| 849 | return _open_brace() | |
| 850 | ||
| 851 | when '}' then | |
| 852 | return _close_brace_or_exit_interpolation() | |
| 853 | ||
| 854 | else | |
| 855 | return TOKEN_PAIR(TOKEN.UNKNOWN, location, "{c}") | |
| 856 | esac | |
| 857 | si | |
| 858 | ||
| 859 | read_operator(c: char mut) -> TOKEN_PAIR => | |
| 860 | read_operator(System.Text.StringBuilder(), c) | |
| 861 | ||
| 862 | // `seed` carries the characters the caller has already consumed on | |
| 863 | // this token's behalf. The numeric-literal scanner can push back only | |
| 864 | // one character, so it hands a `..` following a number over as the | |
| 865 | // single `‥`; seeding with the two dots lets the scan continue into | |
| 866 | // whatever operator characters follow, so `1..<2` and `i..<2` produce | |
| 867 | // the same token. | |
| 868 | read_operator(seed: System.Text.StringBuilder, c: char mut) -> TOKEN_PAIR is | |
| 869 | let _buffer = seed | |
| 870 | let first = if _buffer.length > 0 then _buffer[0] else c fi | |
| 871 | ||
| 872 | // A `.` immediately after a bare `!` or `?` ends the operator: it | |
| 873 | // begins a member access (`x!.foo`, `x?.foo`), not a longer infix | |
| 874 | // operator. Longer operators starting with `!`/`?` (e.g. `!=`) and | |
| 875 | // dot operators (`..`) are unaffected. | |
| 876 | while | |
| 877 | is_operator_char(c) /\ | |
| 878 | !(_buffer.length == 1 /\ (first == '!' \/ first == '?') /\ c == '.') | |
| 879 | do | |
| 880 | _buffer.append(c) | |
| 881 | c = next_char() | |
| 882 | od | |
| 883 | ||
| 884 | prev_char(c) | |
| 885 | ||
| 886 | if _buffer.length < 1 then | |
| 887 | return TOKEN_PAIR(TOKEN.FIRST, location, "") | |
| 888 | fi | |
| 889 | ||
| 890 | let s = _buffer.to_string() | |
| 891 | ||
| 892 | let r mut = TOKEN.OPERATOR | |
| 893 | ||
| 894 | if _operator_tokens.contains_key(s) then | |
| 895 | r = _operator_tokens[s] | |
| 896 | fi | |
| 897 | ||
| 898 | return TOKEN_PAIR(r, location, s) | |
| 899 | si | |
| 900 | ||
| 901 | read_string_fragment() -> (fragment: string, c: char) is | |
| 902 | let fragment = System.Text.StringBuilder() | |
| 903 | ||
| 904 | let c mut = next_char() | |
| 905 | ||
| 906 | while !_end_of_file /\ c != '\n' /\ c != '"' do | |
| 907 | if c == cast char(92) then | |
| 908 | c = read_escape() | |
| 909 | fragment.append(c) | |
| 910 | c = next_char() | |
| 911 | elif c == '{' then | |
| 912 | c = next_char() | |
| 913 | ||
| 914 | if c == '{' then | |
| 915 | fragment.append(c) | |
| 916 | c = next_char() | |
| 917 | else | |
| 918 | prev_char(c) | |
| 919 | c = '{' | |
| 920 | ||
| 921 | break | |
| 922 | fi | |
| 923 | elif c == '}' then | |
| 924 | c = next_char() | |
| 925 | ||
| 926 | if c == '}' then | |
| 927 | fragment.append(c) | |
| 928 | c = next_char() | |
| 929 | else | |
| 930 | _logger.lexer_error(character_location, "unmatched '}}' in string interpolation") | |
| 931 | ||
| 932 | fragment.append('}') | |
| 933 | fi | |
| 934 | else | |
| 935 | fragment.append(c) | |
| 936 | c = next_char() | |
| 937 | fi | |
| 938 | od | |
| 939 | ||
| 940 | if c == '\n' then | |
| 941 | _logger.lexer_error(character_location, "newline in string literal") | |
| 942 | elif _end_of_file then | |
| 943 | _logger.lexer_error(character_location, "end of file in string literal") | |
| 944 | end_of_input_error_locations.add(character_location) | |
| 945 | fi | |
| 946 | ||
| 947 | return (fragment.to_string(), c) | |
| 948 | si | |
| 949 | ||
| 950 | _note_item_start(line: int) is | |
| 951 | _item_starts_line = line != _content_line | |
| 952 | _content_line = line | |
| 953 | si | |
| 954 | ||
| 955 | // Three slashes and not four: a longer run is a divider, not | |
| 956 | // documentation. | |
| 957 | _is_doc_comment(text: string) -> bool => | |
| 958 | _item_starts_line /\ | |
| 959 | text.starts_with("///") /\ | |
| 960 | (text.length == 3 \/ text[3] != '/') | |
| 961 | ||
| 962 | // Consume the rest of a line comment whose `//` has been read, | |
| 963 | // recording it as trivia. | |
| 964 | read_line_comment(comment_location: LOCATION) is | |
| 965 | let buffer = System.Text.StringBuilder("//") | |
| 966 | let c: char mut | |
| 967 | do | |
| 968 | c = next_char() | |
| 969 | if c != '\n' then | |
| 970 | buffer.append(c) | |
| 971 | fi | |
| 972 | ||
| 973 | if !(!_end_of_file/\c!='\n') then | |
| 974 | break | |
| 975 | fi | |
| 976 | od | |
| 977 | ||
| 978 | let text = buffer.to_string().trim_end() | |
| 979 | ||
| 980 | if _is_doc_comment(text) then | |
| 981 | _trivia.add(TRIVIA.DOC_COMMENT(text, comment_location)) | |
| 982 | else | |
| 983 | _trivia.add(TRIVIA.LINE_COMMENT(text, comment_location)) | |
| 984 | fi | |
| 985 | si | |
| 986 | ||
| 987 | // Consume the rest of a block comment whose `/*` has been read, | |
| 988 | // recording it as trivia. | |
| 989 | read_block_comment(comment_location: LOCATION) is | |
| 990 | let buffer = System.Text.StringBuilder("/*") | |
| 991 | let c: char mut | |
| 992 | do | |
| 993 | c = next_char() | |
| 994 | ||
| 995 | if c == '*' then | |
| 996 | c = next_char() | |
| 997 | if c == '/' then | |
| 998 | break | |
| 999 | fi | |
| 1000 | ||
| 1001 | buffer.append('*') | |
| 1002 | fi | |
| 1003 | ||
| 1004 | buffer.append(c) | |
| 1005 | ||
| 1006 | if _end_of_file then | |
| 1007 | break | |
| 1008 | fi | |
| 1009 | od | |
| 1010 | ||
| 1011 | buffer.append("*/") | |
| 1012 | ||
| 1013 | _trivia.add( | |
| 1014 | TRIVIA.BLOCK_COMMENT( | |
| 1015 | buffer.to_string(), | |
| 1016 | comment_location | |
| 1017 | ) | |
| 1018 | ) | |
| 1019 | si | |
| 1020 | ||
| 1021 | // Adjacent string-literal concatenation: while a closed `"..."` | |
| 1022 | // fragment is followed (after whitespace or comments) by another | |
| 1023 | // `"`, fold the next fragment into this token. Stops at the first | |
| 1024 | // fragment that doesn't close with `"` (i.e. starts an | |
| 1025 | // interpolation or hits a newline). Comments between fragments are | |
| 1026 | // recorded as trivia and chained across, so stripping them cannot | |
| 1027 | // change what the token means; only a `;` separates two | |
| 1028 | // otherwise-adjacent literals. | |
| 1029 | // | |
| 1030 | // We treat the whole `"..." "..." "..."` run as a single token. | |
| 1031 | // The token's end position is wherever the final fragment | |
| 1032 | // terminated — we snapshot the cursor's location after each | |
| 1033 | // successful fragment close. When the chain fails (we consume | |
| 1034 | // whitespace then see a non-`"` char), the cursor is past the | |
| 1035 | // whitespace, but we return the snapshot taken at the previous | |
| 1036 | // close so the token's location end isn't smeared across the | |
| 1037 | // following whitespace. The stream stays at the non-`"` char; | |
| 1038 | // the next read_token picks up from there normally. | |
| 1039 | chain_adjacent_string_fragments(initial_fragment: string, initial_terminator: char) -> (fragment: string, c: char, loc: Source.LOCATION) is | |
| 1040 | let buffer = System.Text.StringBuilder() | |
| 1041 | buffer.append(initial_fragment) | |
| 1042 | let c mut = initial_terminator | |
| 1043 | let saved_loc mut = location | |
| 1044 | ||
| 1045 | let done mut = false | |
| 1046 | while !done /\ c == '"' do | |
| 1047 | let peeked = skip_white_space() | |
| 1048 | if peeked == '"' then | |
| 1049 | let (next_fragment, next_terminator) = read_string_fragment() | |
| 1050 | buffer.append(next_fragment) | |
| 1051 | c = next_terminator | |
| 1052 | saved_loc = location | |
| 1053 | elif | |
| 1054 | peeked == '/' /\ | |
| 1055 | (_input.peek() == cast int('/') \/ _input.peek() == cast int('*')) | |
| 1056 | then | |
| 1057 | // A comment counts as the whitespace it sits in, here as | |
| 1058 | // everywhere. The raw peek is sound: the pushback slot is | |
| 1059 | // always empty after skip_white_space hands a char out, so | |
| 1060 | // the reader's next character is the one after the `/`. | |
| 1061 | let comment_location = location | |
| 1062 | ||
| 1063 | _note_item_start(comment_location.start_line) | |
| 1064 | ||
| 1065 | if next_char() == '/' then | |
| 1066 | read_line_comment(comment_location) | |
| 1067 | else | |
| 1068 | read_block_comment(comment_location) | |
| 1069 | fi | |
| 1070 | else | |
| 1071 | prev_char(peeked) | |
| 1072 | done = true | |
| 1073 | fi | |
| 1074 | od | |
| 1075 | ||
| 1076 | return (buffer.to_string(), c, saved_loc) | |
| 1077 | si | |
| 1078 | ||
| 1079 | string_enter() -> TOKEN_PAIR is | |
| 1080 | let (fragment, c) mut = read_string_fragment() | |
| 1081 | ||
| 1082 | let loc: Source.LOCATION mut | |
| 1083 | (fragment, c, loc) = chain_adjacent_string_fragments(fragment, c) | |
| 1084 | ||
| 1085 | // The closing quote settles this before anything else is | |
| 1086 | // consulted: the adjacency lookahead peeks past a literal that | |
| 1087 | // already closed, and can reach the end of the file doing it, | |
| 1088 | // so the end-of-file flag says nothing about this literal once | |
| 1089 | // its quote has been seen. | |
| 1090 | if c == '"' then | |
| 1091 | return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment) | |
| 1092 | elif c == '\n' \/ _end_of_file then | |
| 1093 | _reset_interpolation() | |
| 1094 | ||
| 1095 | return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment) | |
| 1096 | elif c == '{' then | |
| 1097 | _enter_interpolation() | |
| 1098 | ||
| 1099 | return TOKEN_PAIR(TOKEN.ENTER_STRING, loc, fragment) | |
| 1100 | else | |
| 1101 | _logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation") | |
| 1102 | ||
| 1103 | return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment) | |
| 1104 | fi | |
| 1105 | si | |
| 1106 | ||
| 1107 | interpolation_clear() is | |
| 1108 | _reset_interpolation() | |
| 1109 | si | |
| 1110 | ||
| 1111 | _enter_interpolation() is | |
| 1112 | _interpolation_depth = _interpolation_depth + 1 | |
| 1113 | _open_braces.add(0) | |
| 1114 | si | |
| 1115 | ||
| 1116 | _leave_interpolation() is | |
| 1117 | _interpolation_depth = _interpolation_depth - 1 | |
| 1118 | _open_braces.remove_at(_open_braces.count - 1) | |
| 1119 | si | |
| 1120 | ||
| 1121 | _reset_interpolation() is | |
| 1122 | _interpolation_depth = 0 | |
| 1123 | _open_braces.clear() | |
| 1124 | _open_braces.add(0) | |
| 1125 | si | |
| 1126 | ||
| 1127 | _open_brace() -> TOKEN_PAIR is | |
| 1128 | _open_braces[_open_braces.count - 1] = _open_braces[_open_braces.count - 1] + 1 | |
| 1129 | ||
| 1130 | return TOKEN_PAIR(TOKEN.BRACE_OPEN, location, "") | |
| 1131 | si | |
| 1132 | ||
| 1133 | _close_brace_or_exit_interpolation() -> TOKEN_PAIR is | |
| 1134 | let open = _open_braces[_open_braces.count - 1] | |
| 1135 | ||
| 1136 | if open > 0 then | |
| 1137 | _open_braces[_open_braces.count - 1] = open - 1 | |
| 1138 | ||
| 1139 | return TOKEN_PAIR(TOKEN.BRACE_CLOSE, location, "") | |
| 1140 | fi | |
| 1141 | ||
| 1142 | return interpolation_exit('}') | |
| 1143 | si | |
| 1144 | ||
| 1145 | interpolation_exit(c: char mut) -> TOKEN_PAIR is | |
| 1146 | if _interpolation_depth == 0 then | |
| 1147 | _logger.lexer_error(character_location, "unmatched '\"'") | |
| 1148 | next_char() | |
| 1149 | return read_token() | |
| 1150 | fi | |
| 1151 | ||
| 1152 | let fragment mut = "" | |
| 1153 | ||
| 1154 | if c == '}' then | |
| 1155 | (fragment, c) = read_string_fragment() | |
| 1156 | fi | |
| 1157 | ||
| 1158 | let loc: Source.LOCATION mut | |
| 1159 | (fragment, c, loc) = chain_adjacent_string_fragments(fragment, c) | |
| 1160 | ||
| 1161 | // Closing quote first, for the same reason as in string_enter. | |
| 1162 | if c == '"' then | |
| 1163 | _leave_interpolation() | |
| 1164 | ||
| 1165 | return TOKEN_PAIR(TOKEN.EXIT_STRING, loc, fragment) | |
| 1166 | elif c == '\n' \/ _end_of_file then | |
| 1167 | _reset_interpolation() | |
| 1168 | return TOKEN_PAIR(TOKEN.CANCEL_STRING, loc, fragment) | |
| 1169 | elif c == '{' then | |
| 1170 | return TOKEN_PAIR(TOKEN.CONTINUE_STRING, loc, fragment) | |
| 1171 | else | |
| 1172 | _logger.lexer_error(character_location, "unexpected character '{c}' in string interpolation") | |
| 1173 | ||
| 1174 | return TOKEN_PAIR(TOKEN.STRING_LITERAL, loc, fragment) | |
| 1175 | fi | |
| 1176 | si | |
| 1177 | si | |
| 1178 | si |