Appearance
| 1 | namespace Syntax.Parsers is | |
| 2 | use IO.Std | |
| 3 | use Ghul.Disposable | |
| 4 | ||
| 5 | use Logging | |
| 6 | use Source | |
| 7 | ||
| 8 | use Ghul.Pipes | |
| 9 | ||
| 10 | // How a statement terminator was satisfied: a written `;`, a boundary | |
| 11 | // inferred from an end of line, or not at all (an error has been | |
| 12 | // reported). | |
| 13 | enum TERMINATOR is | |
| 14 | WRITTEN, | |
| 15 | INFERRED, | |
| 16 | MISSING, | |
| 17 | si | |
| 18 | ||
| 19 | // Reference type, not a struct: `let use` disposal and the explicit | |
| 20 | // commit()/backtrack()/cancel() calls must act on the same instance, or | |
| 21 | // a speculation can be committed twice (once explicitly, once by the | |
| 22 | // disposer working from a stale copy) — an unbalanced commit underflows | |
| 23 | // the token queue's mark stack and leaves speculation mode stuck on. | |
| 24 | class TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT: Disposable is | |
| 25 | _context: CONTEXT? | |
| 26 | ||
| 27 | is_speculating: bool => _context? | |
| 28 | ||
| 29 | init(context: CONTEXT) is | |
| 30 | _context = context | |
| 31 | _context.tokenizer_speculate() | |
| 32 | si | |
| 33 | ||
| 34 | // _context is cleared before the delegated call, not after: if | |
| 35 | // tokenizer_backtrack/commit throws (a detected speculation loop | |
| 36 | // does throw), the disposer must not run a second time and | |
| 37 | // unbalance the token queue's mark stack. | |
| 38 | backtrack() is | |
| 39 | let context = _context | |
| 40 | _context = null | |
| 41 | context!.tokenizer_backtrack() | |
| 42 | si | |
| 43 | ||
| 44 | backtrack_if_speculating() is | |
| 45 | if _context? then | |
| 46 | backtrack() | |
| 47 | fi | |
| 48 | si | |
| 49 | ||
| 50 | commit() is | |
| 51 | let context = _context | |
| 52 | _context = null | |
| 53 | context!.tokenizer_commit() | |
| 54 | si | |
| 55 | ||
| 56 | commit_if_speculating() is | |
| 57 | if _context? then | |
| 58 | commit() | |
| 59 | fi | |
| 60 | si | |
| 61 | ||
| 62 | cancel() is | |
| 63 | _context = null | |
| 64 | si | |
| 65 | ||
| 66 | dispose() is | |
| 67 | commit_if_speculating() | |
| 68 | si | |
| 69 | si | |
| 70 | ||
| 71 | // Reference type, not a struct — see TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT. | |
| 72 | class TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK: Disposable is | |
| 73 | _context: CONTEXT? | |
| 74 | ||
| 75 | // False for a single, provably-bounded probe whose backtrack must not | |
| 76 | // feed the speculation-loop detector (see TOKEN_LOOKAHEAD.backtrack). | |
| 77 | _check_for_loop: bool | |
| 78 | ||
| 79 | is_speculating: bool => _context? | |
| 80 | ||
| 81 | init(context: CONTEXT) is | |
| 82 | init(context, true) | |
| 83 | si | |
| 84 | ||
| 85 | init(context: CONTEXT, check_for_loop: bool) is | |
| 86 | _context = context | |
| 87 | _check_for_loop = check_for_loop | |
| 88 | _context.tokenizer_speculate() | |
| 89 | si | |
| 90 | ||
| 91 | backtrack() is | |
| 92 | let context = _context | |
| 93 | _context = null | |
| 94 | context!.tokenizer_backtrack(_check_for_loop) | |
| 95 | si | |
| 96 | ||
| 97 | backtrack_if_speculating() is | |
| 98 | if _context? then | |
| 99 | backtrack() | |
| 100 | fi | |
| 101 | si | |
| 102 | ||
| 103 | commit() is | |
| 104 | let context = _context | |
| 105 | _context = null | |
| 106 | context!.tokenizer_commit() | |
| 107 | si | |
| 108 | ||
| 109 | commit_if_speculating() is | |
| 110 | if _context? then | |
| 111 | commit() | |
| 112 | fi | |
| 113 | si | |
| 114 | ||
| 115 | cancel() is | |
| 116 | _context = null | |
| 117 | si | |
| 118 | ||
| 119 | dispose() is | |
| 120 | backtrack_if_speculating() | |
| 121 | si | |
| 122 | si | |
| 123 | ||
| 124 | // Reference type, not a struct — see TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT. | |
| 125 | // | |
| 126 | // Speculate the diagnostics stream and the repeated-error recovery | |
| 127 | // cursor as one unit. The cursor — last_error_location / | |
| 128 | // last_error_message on CONTEXT — drives the recovery in | |
| 129 | // CONTEXT.error(): on an exact-repeat error it drops the duplicate and | |
| 130 | // consumes a token to force progress. A speculative parse that raises | |
| 131 | // an error and is then abandoned has to rewind that cursor along with | |
| 132 | // the diagnostics, or the real parse of the same tokens raises the same | |
| 133 | // error, error() reads it as a repeat, and recovery eats both the | |
| 134 | // diagnostic and a token. Capturing the cursor here makes the rewind | |
| 135 | // automatic: backtrack() (and dispose) restore it, commit() keeps it. | |
| 136 | class DIAGNOSTICS_SPECULATE_THEN_BACKTRACK: Disposable is | |
| 137 | _context: CONTEXT? | |
| 138 | _last_error_location: LOCATION | |
| 139 | _last_error_message: string | |
| 140 | _missing_closer_at: LOCATION? | |
| 141 | ||
| 142 | is_speculating: bool => _context? | |
| 143 | ||
| 144 | init(context: CONTEXT) is | |
| 145 | _context = context | |
| 146 | _last_error_location = context.last_error_location | |
| 147 | _last_error_message = context.last_error_message | |
| 148 | _missing_closer_at = context.missing_closer_at | |
| 149 | _context.logger_speculate() | |
| 150 | si | |
| 151 | ||
| 152 | backtrack() is | |
| 153 | let context = _context | |
| 154 | _context = null | |
| 155 | context!.logger_backtrack() | |
| 156 | context.restore_last_error(_last_error_location, _last_error_message) | |
| 157 | context.missing_closer_at = _missing_closer_at | |
| 158 | si | |
| 159 | ||
| 160 | backtrack_if_speculating() is | |
| 161 | if _context? then | |
| 162 | backtrack() | |
| 163 | fi | |
| 164 | si | |
| 165 | ||
| 166 | commit() is | |
| 167 | let context = _context | |
| 168 | _context = null | |
| 169 | context!.logger_commit() | |
| 170 | si | |
| 171 | ||
| 172 | commit_if_speculating() is | |
| 173 | if _context? then | |
| 174 | commit() | |
| 175 | fi | |
| 176 | si | |
| 177 | ||
| 178 | cancel() is | |
| 179 | _context = null | |
| 180 | si | |
| 181 | ||
| 182 | dispose() is | |
| 183 | backtrack_if_speculating() | |
| 184 | si | |
| 185 | si | |
| 186 | ||
| 187 | // Holds a construct's enders open for the span between the parser | |
| 188 | // consuming its opening keyword and expecting its closing one. Used as | |
| 189 | // `let use`, so they are released however that span is left - the | |
| 190 | // return, or one of the recovery unwinds. | |
| 191 | class OPEN_CONSTRUCT: Disposable is | |
| 192 | _context: CONTEXT? | |
| 193 | ||
| 194 | init(context: CONTEXT, enders: Collections.LIST[Lexical.TOKEN]) is | |
| 195 | _context = context | |
| 196 | ||
| 197 | context.push_awaited_enders(enders) | |
| 198 | si | |
| 199 | ||
| 200 | dispose() is | |
| 201 | let context = _context | |
| 202 | _context = null | |
| 203 | ||
| 204 | if context? then | |
| 205 | context.pop_awaited_enders() | |
| 206 | fi | |
| 207 | si | |
| 208 | si | |
| 209 | ||
| 210 | class CONTEXT is | |
| 211 | _repeated_error_count_at_eof: int | |
| 212 | ||
| 213 | last_error_location: LOCATION | |
| 214 | last_error_message: string | |
| 215 | ||
| 216 | // Where a construct last failed to find its closing keyword, and | |
| 217 | // nothing if the last one closed cleanly. See expect_closer. | |
| 218 | missing_closer_at: LOCATION? public | |
| 219 | ||
| 220 | // The tokens that would end the body each parser currently in this | |
| 221 | // parse is reading - a construct's own closing keyword, and the | |
| 222 | // keywords that start its next clause. Each construct parser holds | |
| 223 | // its own open across the span between its opening keyword and its | |
| 224 | // closing one, so this is the recursive descent's own stack made | |
| 225 | // legible rather than a second copy of it kept in step by hand. | |
| 226 | // | |
| 227 | // Error recovery reads it to tell a keyword that some construct | |
| 228 | // still being parsed is waiting for - where a skip has to stop and | |
| 229 | // hand the token up - from one that ends nothing open, which is | |
| 230 | // debris from a construct the parser never entered and can be | |
| 231 | // skipped past. | |
| 232 | // | |
| 233 | // Nothing here counts anything, and deliberately so. Counting how | |
| 234 | // deeply nested the source is counts the very thing the mistake | |
| 235 | // being recovered from has broken: brackets in code that is being | |
| 236 | // edited are not balanced, which is what makes it code that needs | |
| 237 | // recovering. What a parser is waiting for stays true whatever the | |
| 238 | // source does next. | |
| 239 | _awaited_enders: Collections.LIST[Collections.LIST[Lexical.TOKEN]] | |
| 240 | ||
| 241 | // The indentation of the line each awaiting construct opened on, in | |
| 242 | // step with _awaited_enders. Read only by error recovery, to tell | |
| 243 | // which of two constructs a misplaced closer was written for. | |
| 244 | _awaited_indents: Collections.LIST[int] | |
| 245 | ||
| 246 | // Where each error reported with the parser at end of input was | |
| 247 | // located. A speculative report that is later rolled back stays here, | |
| 248 | // which is harmless: this lists what could be an early end, and the | |
| 249 | // reports that stand are what a caller checks against it. | |
| 250 | end_of_input_error_locations: Collections.LIST[LOCATION] | |
| 251 | ||
| 252 | // How many errors have been reported with the parser anywhere but at | |
| 253 | // end of input, rolled-back speculative reports included. | |
| 254 | errors_away_from_end_of_input: int | |
| 255 | ||
| 256 | // The line the parser last reported on, rolled-back speculative | |
| 257 | // reports included, so a caller weighing two readings of the same | |
| 258 | // tokens can tell a mistake inside what it was reading from one | |
| 259 | // far beyond it. | |
| 260 | last_error_line: int public | |
| 261 | ||
| 262 | allow_tuple_element: bool public | |
| 263 | ||
| 264 | // Set by the statement parser for the one expression parse where a | |
| 265 | // named function definition is legal - a statement head whose shape | |
| 266 | // can only be one. The secondary parser clears it as soon as it | |
| 267 | // takes the name, so a literal written anywhere inside that | |
| 268 | // statement, the function body included, is still anonymous. | |
| 269 | allow_nested_function: bool public | |
| 270 | ||
| 271 | // True while the statement being parsed is one of a file's bare | |
| 272 | // top-level statements, where a named function is a global | |
| 273 | // definition and its argument types are required. A lambda body | |
| 274 | // written inside one is an ordinary body again, and clears it. | |
| 275 | in_top_level_statements: bool public | |
| 276 | ||
| 277 | in_trait: bool public | |
| 278 | ||
| 279 | in_global_function: bool public | |
| 280 | in_classy: bool public | |
| 281 | in_member: bool public | |
| 282 | ||
| 283 | // Depth of enclosing `namespace ... is ... si` blocks. Zero only at | |
| 284 | // the file root, where a file with no namespace may carry bare | |
| 285 | // top-level statements collected into a synthesised entry point. | |
| 286 | // GLOBAL_LIST (reused as the namespace body parser) consults this to | |
| 287 | // confine that handling to depth 0; inside any namespace it behaves | |
| 288 | // exactly as before. | |
| 289 | namespace_depth: int public | |
| 290 | ||
| 291 | // True only while parsing the `[ ... ]` type-parameter list of a | |
| 292 | // generic type declaration. Distinguishes a type-parameter | |
| 293 | // declaration from a named-tuple-element type (both produce | |
| 294 | // NAMED_TUPLE_ELEMENT via the same parse path) so the `out` / | |
| 295 | // `in` variance modifier is recognised only on a type parameter. | |
| 296 | in_type_parameters: bool public | |
| 297 | ||
| 298 | // True only while parsing any `[ ... ]` list of type parameters — | |
| 299 | // a type definition's header or a function or method's generic | |
| 300 | // arguments. This is wider than in_type_parameters, which covers | |
| 301 | // only type definitions, and it is what decides whether a second | |
| 302 | // `/\`-joined bound after a bound belongs on a type parameter. | |
| 303 | in_generic_parameter_list: bool public | |
| 304 | ||
| 305 | // True only at the head of one element of a type-parameter list, | |
| 306 | // and cleared as soon as that element's own name has been read. | |
| 307 | // The `..` argument-pack marker is postfix on the parameter | |
| 308 | // itself, so this keeps it from being taken as a marker on a | |
| 309 | // type named inside the same element - a bound, say. | |
| 310 | at_generic_parameter_head: bool public | |
| 311 | ||
| 312 | // True only while parsing the formal-argument list of an | |
| 313 | // `init` method. The `..` splice marker is permitted there | |
| 314 | // (it's expanded by the primary-constructor rewrite); anywhere | |
| 315 | // else `..` in a variable-list position is a syntax error. | |
| 316 | in_init_arguments: bool public | |
| 317 | ||
| 318 | // True only while parsing the primary-ctor parameter list of a | |
| 319 | // class or struct header. The variable parser consults this to | |
| 320 | // decide whether to read a trailing modifier list on each | |
| 321 | // parameter — in other variable-parsing contexts (`let`, formal | |
| 322 | // args, secondary inits) it leaves the modifier slot alone, so | |
| 323 | // a stray `public` / `init` there falls through to the caller's | |
| 324 | // normal "unexpected token" handling. The modifier parser itself | |
| 325 | // matches `init` unconditionally; gating happens only here at | |
| 326 | // the variable-parser layer. | |
| 327 | in_primary_ctor_params: bool public | |
| 328 | ||
| 329 | // True only while parsing the formal-argument list of a | |
| 330 | // function or method definition. The variable parser consults | |
| 331 | // this to decide whether an `@` before a parameter starts an | |
| 332 | // attribute pragma — elsewhere (`let`, primary-ctor headers, | |
| 333 | // variant field lists) a stray `@` falls through to the | |
| 334 | // caller's normal "unexpected token" handling. | |
| 335 | in_formal_arguments: bool public | |
| 336 | ||
| 337 | // True only while parsing a function or method definition's | |
| 338 | // declared return type, after the `->`. The argument-pack | |
| 339 | // marker is legal there on the parameter of a function type - | |
| 340 | // `-> T.. -> U` says the function returns an N-ary one - as it | |
| 341 | // is on a formal's own type. | |
| 342 | in_return_type: bool public | |
| 343 | ||
| 344 | // True only while parsing a pattern that can fail to match — | |
| 345 | // an `if let` / `while let` clause or a `case` `when` arm. The | |
| 346 | // destructure parser consults this to reject the `~` match | |
| 347 | // marker everywhere a leaf can only ever bind: a plain `let`, | |
| 348 | // a formal argument, a lambda parameter. Defaulting to false | |
| 349 | // means a position added later rejects the marker until it | |
| 350 | // says otherwise. | |
| 351 | in_refutable_pattern: bool public | |
| 352 | ||
| 353 | // intentation is not part of the syntax, but it is used for | |
| 354 | // for error recovery: we may want to skip over a block of code | |
| 355 | // out to a given indentation level | |
| 356 | global_indent: int public | |
| 357 | member_indent: int public | |
| 358 | ||
| 359 | tokenizer: Lexical.TOKEN_LOOKAHEAD | |
| 360 | logger: Logger | |
| 361 | ||
| 362 | location: LOCATION => current.location | |
| 363 | ||
| 364 | is_end_of_file: bool | |
| 365 | ||
| 366 | current: Lexical.TOKEN_PAIR | |
| 367 | ||
| 368 | // End of the token before `current` - where an inferred statement | |
| 369 | // terminator belongs, since that is the position the `;` would have | |
| 370 | // been written at. Tracked here rather than taken from the caller's | |
| 371 | // `end`: several callers pass `context.location`, which after a line | |
| 372 | // break is the *next* line's first token, so a hint placed there | |
| 373 | // lands on the following construct. | |
| 374 | // | |
| 375 | // Maintained wherever `current` moves. A backtrack cannot recompute | |
| 376 | // it from the restored token - that token's own location is where | |
| 377 | // `current` is, not where the one before it ended - so it is saved | |
| 378 | // on speculate and restored on backtrack, alongside the token | |
| 379 | // stream it belongs to. | |
| 380 | previous_end: LOCATION | |
| 381 | ||
| 382 | // previous_end as it stood at each open speculation, innermost last. | |
| 383 | _previous_end_marks: Collections.STACK[LOCATION] | |
| 384 | ||
| 385 | current_token: Lexical.TOKEN => current.token | |
| 386 | ||
| 387 | current_string: string? => current.value_string | |
| 388 | ||
| 389 | current_token_name: string is | |
| 390 | let result mut = Lexical.TOKEN_NAMES[current_token] | |
| 391 | ||
| 392 | if current_string? /\ current_string !~ result then | |
| 393 | result = "{result} {current_string}" | |
| 394 | fi | |
| 395 | ||
| 396 | return result | |
| 397 | si | |
| 398 | ||
| 399 | init(tokenizer: Lexical.TOKEN_LOOKAHEAD, logger: Logger) is | |
| 400 | self.tokenizer = tokenizer | |
| 401 | self.logger = logger | |
| 402 | ||
| 403 | _previous_end_marks = Collections.STACK[LOCATION]() | |
| 404 | _awaited_enders = Collections.LIST[Collections.LIST[Lexical.TOKEN]]() | |
| 405 | _awaited_indents = Collections.LIST[int]() | |
| 406 | end_of_input_error_locations = Collections.LIST[LOCATION]() | |
| 407 | ||
| 408 | current = tokenizer.read_token() | |
| 409 | previous_end = current.location | |
| 410 | ||
| 411 | // next_token() sets this on every read after the first; the | |
| 412 | // first-token read here has to match. | |
| 413 | is_end_of_file = current.token == Lexical.TOKEN.END_OF_INPUT | |
| 414 | ||
| 415 | last_error_location = self.location | |
| 416 | last_error_message = "" | |
| 417 | si | |
| 418 | ||
| 419 | tokenizer_speculate_then_commit() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT => | |
| 420 | TOKEN_LOOKAHEAD_SPECULATE_THEN_COMMIT(self) | |
| 421 | ||
| 422 | tokenizer_speculate_then_backtrack() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK => | |
| 423 | TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK(self) | |
| 424 | ||
| 425 | // A single, provably-bounded probe: its backtrack does not feed the | |
| 426 | // speculation-loop detector. | |
| 427 | tokenizer_speculate_then_backtrack_bounded() -> TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK => | |
| 428 | TOKEN_LOOKAHEAD_SPECULATE_THEN_BACKTRACK(self, false) | |
| 429 | ||
| 430 | logger_speculate_then_commit() -> LOGGER_SPECULATE_THEN_COMMIT => logger.speculate_then_commit() | |
| 431 | ||
| 432 | diagnostics_speculate_then_backtrack() -> DIAGNOSTICS_SPECULATE_THEN_BACKTRACK => | |
| 433 | DIAGNOSTICS_SPECULATE_THEN_BACKTRACK(self) | |
| 434 | ||
| 435 | tokenizer_speculate() is | |
| 436 | _previous_end_marks.push(previous_end) | |
| 437 | ||
| 438 | tokenizer.speculate() | |
| 439 | si | |
| 440 | ||
| 441 | tokenizer_commit() is | |
| 442 | // The speculated tokens are kept, so the position reached | |
| 443 | // inside them is the position now. | |
| 444 | _pop_previous_end_mark() | |
| 445 | ||
| 446 | tokenizer.commit() | |
| 447 | si | |
| 448 | ||
| 449 | tokenizer_backtrack() is | |
| 450 | current = tokenizer.backtrack() | |
| 451 | previous_end = _pop_previous_end_mark() ?? current.location | |
| 452 | si | |
| 453 | ||
| 454 | tokenizer_backtrack(check_for_loop: bool) is | |
| 455 | current = tokenizer.backtrack(check_for_loop) | |
| 456 | previous_end = _pop_previous_end_mark() ?? current.location | |
| 457 | si | |
| 458 | ||
| 459 | // Null when the stack is empty, which happens only if a commit or | |
| 460 | // backtrack is reached without its speculate - a caller bug the | |
| 461 | // tokenizer's own mark stack reports; the fallback keeps this from | |
| 462 | // adding a second failure on top of it. | |
| 463 | _pop_previous_end_mark() -> LOCATION? is | |
| 464 | if _previous_end_marks.count == 0 then | |
| 465 | return null | |
| 466 | fi | |
| 467 | ||
| 468 | return _previous_end_marks.pop() | |
| 469 | si | |
| 470 | ||
| 471 | logger_speculate() is | |
| 472 | logger.speculate() | |
| 473 | si | |
| 474 | ||
| 475 | logger_commit() is | |
| 476 | logger.commit() | |
| 477 | si | |
| 478 | ||
| 479 | logger_backtrack() -> DIAGNOSTICS_STATE => logger.roll_back() | |
| 480 | ||
| 481 | // Rewind the repeated-error recovery cursor read by error() below. | |
| 482 | // Called by DIAGNOSTICS_SPECULATE_THEN_BACKTRACK when a speculative | |
| 483 | // parse is abandoned; see that class for why the cursor has to move | |
| 484 | // in step with the diagnostics. | |
| 485 | restore_last_error(location: LOCATION, message: string) is | |
| 486 | last_error_location = location | |
| 487 | last_error_message = message | |
| 488 | si | |
| 489 | expect_format_specifier() is | |
| 490 | tokenizer.expect_format_specifier() | |
| 491 | si | |
| 492 | ||
| 493 | next_token() -> bool is | |
| 494 | previous_end = current.location | |
| 495 | current = tokenizer.read_token() | |
| 496 | ||
| 497 | is_end_of_file = current.token == Lexical.TOKEN.END_OF_INPUT | |
| 498 | ||
| 499 | return is_end_of_file | |
| 500 | si | |
| 501 | ||
| 502 | next_token(token: Lexical.TOKEN, message: string) -> bool is | |
| 503 | if expect_token(token, message) then | |
| 504 | next_token() | |
| 505 | return true | |
| 506 | fi | |
| 507 | return false | |
| 508 | si | |
| 509 | ||
| 510 | next_token(tokens: Collections.List[Lexical.TOKEN], message: string) -> bool is | |
| 511 | if expect_token(tokens, message) then | |
| 512 | next_token() | |
| 513 | return true | |
| 514 | fi | |
| 515 | return false | |
| 516 | si | |
| 517 | ||
| 518 | next_token(token: Lexical.TOKEN) -> bool => next_token(token, "syntax error") | |
| 519 | next_token(tokens: Collections.List[Lexical.TOKEN]) -> bool => next_token(tokens, "syntax error") | |
| 520 | ||
| 521 | // True when the grammar expects a statement terminator here and the | |
| 522 | // line break before the current token can stand in for it: the | |
| 523 | // current token opened a new source line. End of file ends the last | |
| 524 | // line as well as the file, so it stands in too. | |
| 525 | at_inferred_terminator: bool => | |
| 526 | current.first_on_line \/ is_end_of_file | |
| 527 | ||
| 528 | // Whether the current token can continue the construct that began | |
| 529 | // at `anchor`, at a point where the grammar could equally end that | |
| 530 | // construct and hand the token to an enclosing one. On the | |
| 531 | // anchor's own line it always can. On a later line it can only | |
| 532 | // when that line is indented at least as far as the anchor's: a | |
| 533 | // token dedented past the construct it would extend closes an | |
| 534 | // enclosing one instead, as the `else` of the `if` around a bare | |
| 535 | // `assert` does. | |
| 536 | continues(anchor: LOCATION) -> bool => | |
| 537 | !current.first_on_line \/ | |
| 538 | current.location.start_column >= tokenizer.indent_of_line(anchor.start_line) | |
| 539 | ||
| 540 | // Accept a statement terminator: a written `;` is consumed, and | |
| 541 | // where one is missing an end-of-line boundary stands in for it. | |
| 542 | // Callers only reach this at points where the grammar could accept | |
| 543 | // a `;`, which is what keeps the inference parser-driven. | |
| 544 | accept_terminator(message: string) -> TERMINATOR is | |
| 545 | if current_token == Lexical.TOKEN.SEMICOLON then | |
| 546 | let semicolon = current | |
| 547 | next_token() | |
| 548 | note_written_terminator(semicolon) | |
| 549 | return TERMINATOR.WRITTEN | |
| 550 | fi | |
| 551 | ||
| 552 | if at_inferred_terminator then | |
| 553 | note_inferred_terminator() | |
| 554 | ||
| 555 | return TERMINATOR.INFERRED | |
| 556 | fi | |
| 557 | ||
| 558 | expect_token(Lexical.TOKEN.SEMICOLON, message) | |
| 559 | return TERMINATOR.MISSING | |
| 560 | si | |
| 561 | ||
| 562 | accept_terminator() -> TERMINATOR => accept_terminator("syntax error") | |
| 563 | ||
| 564 | // Record a statement boundary that no `;` marks, for the terminator | |
| 565 | // inlay. The hint goes after the last token actually consumed: the | |
| 566 | // current token is the next line's first, and the boundary is behind | |
| 567 | // it. Every boundary a written `;` would be reported redundant at | |
| 568 | // gets one, so the two are exact complements - which includes the | |
| 569 | // boundaries the grammar never demanded a terminator at, after a | |
| 570 | // compound statement and at the end of a statement list. | |
| 571 | note_inferred_terminator() is | |
| 572 | if at_inferred_terminator then | |
| 573 | logger.note_inferred_terminator(previous_end) | |
| 574 | fi | |
| 575 | si | |
| 576 | ||
| 577 | // A written `;` at the end of a source line adds nothing the | |
| 578 | // line break would not infer: nothing reads the terminator for | |
| 579 | // meaning, a body's tail included, which is judged by its type. | |
| 580 | // Report them so the terminator-free style has an enforcing | |
| 581 | // warning; the slug is suppressed by default and enabled with | |
| 582 | // `--warn redundant-semicolon`. Neither does the last one in a | |
| 583 | // file, since end of file ends a line as well as the file. | |
| 584 | // | |
| 585 | // The one exemption is a `;` between two string literals: without | |
| 586 | // it the tokenizer chains the fragments into a single literal, so | |
| 587 | // it separates rather than terminates and removing it changes the | |
| 588 | // program. That the current token opened a new line makes no | |
| 589 | // difference — fragment chaining crosses line breaks. | |
| 590 | note_written_terminator(semicolon: Lexical.TOKEN_PAIR) is | |
| 591 | if !at_inferred_terminator then | |
| 592 | return | |
| 593 | fi | |
| 594 | ||
| 595 | if | |
| 596 | semicolon.follows_string /\ | |
| 597 | (current_token == Lexical.TOKEN.STRING_LITERAL \/ | |
| 598 | current_token == Lexical.TOKEN.ENTER_STRING) | |
| 599 | then | |
| 600 | return | |
| 601 | fi | |
| 602 | ||
| 603 | logger.warn(semicolon.location, "redundant-semicolon", "';' is redundant here") | |
| 604 | si | |
| 605 | location_and_next() -> LOCATION is | |
| 606 | let result = location | |
| 607 | next_token() | |
| 608 | return result | |
| 609 | si | |
| 610 | ||
| 611 | skip_token(tokens: Collections.List[Lexical.TOKEN], message: string) is | |
| 612 | let start = location | |
| 613 | ||
| 614 | do | |
| 615 | if is_end_of_file then | |
| 616 | error(start::location, "{message}: expected {Lexical.TOKEN_NAMES[current_token]}") | |
| 617 | return | |
| 618 | elif tokens |> any(t => t == current_token) then | |
| 619 | error(start::location, "{message}: expected {Lexical.TOKEN_NAMES[current_token]}") | |
| 620 | next_token() | |
| 621 | return | |
| 622 | else | |
| 623 | next_token() | |
| 624 | fi | |
| 625 | od | |
| 626 | si | |
| 627 | ||
| 628 | skip_token(token: Lexical.TOKEN, message: string) is | |
| 629 | let t = Collections.LIST[Lexical.TOKEN]() | |
| 630 | t.add(token) | |
| 631 | skip_token(t, message) | |
| 632 | si | |
| 633 | ||
| 634 | expect_token(token: Lexical.TOKEN, message: string) -> bool is | |
| 635 | if current_token != token then | |
| 636 | if current_token != Lexical.TOKEN.CANCEL_STRING then | |
| 637 | error(location, "{message}: expected {Lexical.TOKEN_NAMES[token]} but found {current_token_name}") | |
| 638 | fi | |
| 639 | ||
| 640 | return false | |
| 641 | else | |
| 642 | return true | |
| 643 | fi | |
| 644 | si | |
| 645 | ||
| 646 | expect_token(token: Lexical.TOKEN) -> bool => expect_token(token, "syntax error") | |
| 647 | expect_token(tokens: Collections.List[Lexical.TOKEN], message: string) -> bool is | |
| 648 | if !(tokens |> any(t => t == current_token)) then | |
| 649 | if current_token != Lexical.TOKEN.CANCEL_STRING then | |
| 650 | error(location, "{message}: expected {Lexical.TOKEN_NAMES[tokens]} but found {current_token_name}") | |
| 651 | fi | |
| 652 | ||
| 653 | return false | |
| 654 | else | |
| 655 | return true | |
| 656 | fi | |
| 657 | si | |
| 658 | ||
| 659 | expect_token(tokens: Collections.List[Lexical.TOKEN]) -> bool => | |
| 660 | expect_token(tokens, "syntax error") | |
| 661 | ||
| 662 | // The closing keyword of a construct, which is a different thing | |
| 663 | // from any other expected token: every construct that was open when | |
| 664 | // one goes missing wants its own closer at the same token, and each | |
| 665 | // of them reports it. The `while` wanting `od`, the body wanting | |
| 666 | // `si` and the definition list wanting a definition are one mistake | |
| 667 | // described three times, and only the first of them - the innermost, | |
| 668 | // which is the one that knows which closer is actually missing - is | |
| 669 | // worth saying. So the first report at a token marks it, and the | |
| 670 | // constructs unwinding through the same token afterwards stay quiet. | |
| 671 | // | |
| 672 | // The mark is cleared by any closer that is found, so it never | |
| 673 | // reaches a later, unrelated failure; it is keyed on the location | |
| 674 | // as well, so it could not anyway. | |
| 675 | expect_closer(token: Lexical.TOKEN, message: string) -> bool is | |
| 676 | if current_token == token then | |
| 677 | missing_closer_at = null | |
| 678 | ||
| 679 | next_token() | |
| 680 | ||
| 681 | return true | |
| 682 | fi | |
| 683 | ||
| 684 | if current_token == Lexical.TOKEN.CANCEL_STRING then | |
| 685 | return false | |
| 686 | fi | |
| 687 | ||
| 688 | let at = location | |
| 689 | ||
| 690 | if !(missing_closer_at? /\ missing_closer_at =~ at) then | |
| 691 | error(at, "{message}: expected {Lexical.TOKEN_NAMES[token]} but found {current_token_name}") | |
| 692 | fi | |
| 693 | ||
| 694 | if _is_misplaced_closer_for(token) then | |
| 695 | missing_closer_at = null | |
| 696 | ||
| 697 | next_token() | |
| 698 | ||
| 699 | return true | |
| 700 | fi | |
| 701 | ||
| 702 | missing_closer_at = at | |
| 703 | ||
| 704 | return false | |
| 705 | si | |
| 706 | ||
| 707 | expect_closer(token: Lexical.TOKEN) -> bool => expect_closer(token, "syntax error") | |
| 708 | ||
| 709 | // Whether the closing keyword found where the innermost construct's | |
| 710 | // own was expected is that construct's closer, mistyped, rather than | |
| 711 | // an enclosing construct's closer arriving early. It is the | |
| 712 | // innermost's when it is not dedented past the innermost's own line | |
| 713 | // and either no enclosing construct is waiting for it, or the | |
| 714 | // nearest one that is waiting opened on a line indented less deeply | |
| 715 | // than the keyword is written: a closer is written level with the | |
| 716 | // construct it closes. Consuming it keeps the statements after | |
| 717 | // it in the body they were written in, where leaving it would end | |
| 718 | // that body and hand them to the member list. | |
| 719 | _is_misplaced_closer_for(expected: Lexical.TOKEN) -> bool is | |
| 720 | let token = current_token | |
| 721 | ||
| 722 | if !_is_construct_closer(token) then | |
| 723 | return false | |
| 724 | fi | |
| 725 | ||
| 726 | // A construct that releases its enders before expecting its | |
| 727 | // closer is no longer on the stack, so the innermost frame is | |
| 728 | // its own only while it still awaits the expected closer. | |
| 729 | let top = _awaited_enders.count - 1 | |
| 730 | let is_own = top >= 0 /\ _awaited_enders[top] |> any(t => t == expected) | |
| 731 | ||
| 732 | // Dedented past the line the construct opened on, it closes | |
| 733 | // something further out. | |
| 734 | if is_own /\ current.first_on_line /\ current.location.start_column < _awaited_indents[top] then | |
| 735 | return false | |
| 736 | fi | |
| 737 | ||
| 738 | let outer mut = if is_own then top - 1 else top fi | |
| 739 | ||
| 740 | while outer >= 0 do | |
| 741 | if _awaited_enders[outer] |> any(t => t == token) then | |
| 742 | return current.first_on_line /\ current.location.start_column > _awaited_indents[outer] | |
| 743 | fi | |
| 744 | ||
| 745 | outer = outer - 1 | |
| 746 | od | |
| 747 | ||
| 748 | return true | |
| 749 | si | |
| 750 | ||
| 751 | // Whether the current token is a closing or clause keyword that no | |
| 752 | // construct still open can have been written for: the innermost is not | |
| 753 | // waiting for it, and it stands on its own line indented deeper | |
| 754 | // than the line every open construct that would take it opened on. | |
| 755 | // A closer is written level with its construct, so this one is | |
| 756 | // debris - a duplicate, or one left behind by an edit - and a | |
| 757 | // statement list skips it rather than ending there. | |
| 758 | at_debris_closer: bool is | |
| 759 | let token = current_token | |
| 760 | ||
| 761 | if !(_is_construct_closer(token) \/ is_clause_keyword(token)) \/ !current.first_on_line then | |
| 762 | return false | |
| 763 | fi | |
| 764 | ||
| 765 | let innermost = _awaited_enders.count - 1 | |
| 766 | ||
| 767 | if innermost < 0 \/ _awaited_enders[innermost] |> any(t => t == token) then | |
| 768 | return false | |
| 769 | fi | |
| 770 | ||
| 771 | let column = current.location.start_column | |
| 772 | ||
| 773 | if column <= _awaited_indents[innermost] then | |
| 774 | return false | |
| 775 | fi | |
| 776 | ||
| 777 | let outer mut = innermost - 1 | |
| 778 | ||
| 779 | while outer >= 0 do | |
| 780 | if column <= _awaited_indents[outer] /\ _awaited_enders[outer] |> any(t => t == token) then | |
| 781 | return false | |
| 782 | fi | |
| 783 | ||
| 784 | outer = outer - 1 | |
| 785 | od | |
| 786 | ||
| 787 | return true | |
| 788 | si | |
| 789 | ||
| 790 | // A keyword that starts the next clause of a construct. Where a | |
| 791 | // program means one, the construct it belongs to is the innermost | |
| 792 | // open one and is waiting for it, so one that nothing open awaits | |
| 793 | // is debris like a stray closer - a pasted duplicate, say. | |
| 794 | is_clause_keyword(token: Lexical.TOKEN) -> bool => | |
| 795 | token == Lexical.TOKEN.ELSE \/ | |
| 796 | token == Lexical.TOKEN.ELIF \/ | |
| 797 | token == Lexical.TOKEN.WHEN \/ | |
| 798 | token == Lexical.TOKEN.CATCH \/ | |
| 799 | token == Lexical.TOKEN.FINALLY | |
| 800 | ||
| 801 | _is_construct_closer(token: Lexical.TOKEN) -> bool => | |
| 802 | token == Lexical.TOKEN.FI \/ | |
| 803 | token == Lexical.TOKEN.OD \/ | |
| 804 | token == Lexical.TOKEN.ESAC \/ | |
| 805 | token == Lexical.TOKEN.YRT \/ | |
| 806 | token == Lexical.TOKEN.LAV \/ | |
| 807 | token == Lexical.TOKEN.SI | |
| 808 | ||
| 809 | // Hold `enders` open until the returned value is disposed. | |
| 810 | open_construct(enders: Collections.LIST[Lexical.TOKEN]) -> OPEN_CONSTRUCT => | |
| 811 | OPEN_CONSTRUCT(self, enders) | |
| 812 | ||
| 813 | open_construct(ender: Lexical.TOKEN) -> OPEN_CONSTRUCT => | |
| 814 | OPEN_CONSTRUCT(self, Collections.LIST[Lexical.TOKEN]([ender])) | |
| 815 | ||
| 816 | push_awaited_enders(enders: Collections.LIST[Lexical.TOKEN]) is | |
| 817 | _awaited_enders.add(enders) | |
| 818 | _awaited_indents.add(tokenizer.indent_of_line(previous_end.start_line)) | |
| 819 | si | |
| 820 | ||
| 821 | pop_awaited_enders() is | |
| 822 | if _awaited_enders.count > 0 then | |
| 823 | _awaited_enders.remove_at(_awaited_enders.count - 1) | |
| 824 | _awaited_indents.remove_at(_awaited_indents.count - 1) | |
| 825 | fi | |
| 826 | si | |
| 827 | ||
| 828 | // Whether some construct still being parsed would end at this token. | |
| 829 | // True of the innermost as well as any further out, so a skip that | |
| 830 | // stops here hands the token to whichever of them claims it first. | |
| 831 | is_awaited_ender(token: Lexical.TOKEN) -> bool => | |
| 832 | _awaited_enders |> any(enders => enders |> any(t => t == token)) | |
| 833 | ||
| 834 | // Debris: a statement construct's closing or clause keyword sitting | |
| 835 | // where a definition should be, after a construct has already been | |
| 836 | // reported as unclosed. Every keyword left over by that failure | |
| 837 | // arrives here one at a time, and reporting each as a definition | |
| 838 | // that is not one describes the same mistake once per keyword - the | |
| 839 | // shape a reader sees as a file-long cascade. | |
| 840 | // | |
| 841 | // Only debris is skipped: without a missing closer behind it, a | |
| 842 | // stray `fi` is reported as it always was, and the mark is cleared | |
| 843 | // by the next closer that is found, so the silence lasts exactly as | |
| 844 | // long as the unwinding does. | |
| 845 | // | |
| 846 | // `si` closes a namespace as well as a body, so the caller says | |
| 847 | // whether one is debris where it stands. | |
| 848 | at_stray_construct_closer(include_si: bool) -> bool is | |
| 849 | if !missing_closer_at? then | |
| 850 | return false | |
| 851 | fi | |
| 852 | ||
| 853 | let token = current_token | |
| 854 | ||
| 855 | if include_si /\ token == Lexical.TOKEN.SI then | |
| 856 | return true | |
| 857 | fi | |
| 858 | ||
| 859 | token == Lexical.TOKEN.FI \/ | |
| 860 | token == Lexical.TOKEN.OD \/ | |
| 861 | token == Lexical.TOKEN.ESAC \/ | |
| 862 | token == Lexical.TOKEN.YRT \/ | |
| 863 | token == Lexical.TOKEN.LAV \/ | |
| 864 | token == Lexical.TOKEN.CATCH \/ | |
| 865 | token == Lexical.TOKEN.FINALLY \/ | |
| 866 | token == Lexical.TOKEN.ELSE \/ | |
| 867 | token == Lexical.TOKEN.ELIF \/ | |
| 868 | token == Lexical.TOKEN.WHEN \/ | |
| 869 | token == Lexical.TOKEN.THEN | |
| 870 | si | |
| 871 | ||
| 872 | warn(location: LOCATION, message: string) is | |
| 873 | logger.warn(location, message) | |
| 874 | si | |
| 875 | ||
| 876 | // The repeated-error recovery cursor: true when `location` and | |
| 877 | // `message` are new and should be reported, having updated the | |
| 878 | // cursor to match; false when this is an exact repeat of the | |
| 879 | // last error, having already taken the recovery action (consume | |
| 880 | // a token, or count towards the end-of-file bail-out). | |
| 881 | _should_report_error(location: LOCATION, message: string) -> bool is | |
| 882 | if location !~ last_error_location \/ message !~ last_error_message then | |
| 883 | _repeated_error_count_at_eof = 0 | |
| 884 | ||
| 885 | last_error_location = location | |
| 886 | last_error_message = message | |
| 887 | ||
| 888 | return true | |
| 889 | elif !is_end_of_file then | |
| 890 | next_token() | |
| 891 | else | |
| 892 | _repeated_error_count_at_eof = _repeated_error_count_at_eof + 1 | |
| 893 | ||
| 894 | if (_repeated_error_count_at_eof > 10) then | |
| 895 | throw Compiler.PARSE_EXCEPTION("repeated errors at end of file") | |
| 896 | fi | |
| 897 | fi | |
| 898 | ||
| 899 | return false | |
| 900 | si | |
| 901 | ||
| 902 | error(location: LOCATION, message: string) is | |
| 903 | if _should_report_error(location, message) then | |
| 904 | logger.error(location, message) | |
| 905 | _note_if_at_end_of_input(location) | |
| 906 | fi | |
| 907 | si | |
| 908 | ||
| 909 | // Same as error(location, message), plus a related location. | |
| 910 | error(location: LOCATION, message: string, related_location: LOCATION, related_message: string) is | |
| 911 | if _should_report_error(location, message) then | |
| 912 | logger.error(location, message, related_location, related_message) | |
| 913 | _note_if_at_end_of_input(location) | |
| 914 | fi | |
| 915 | si | |
| 916 | ||
| 917 | // An error reported with the parser at end of input is one more input | |
| 918 | // could have answered: the construct being read was still open, or | |
| 919 | // the value it demanded had not been written yet. Recorded so a | |
| 920 | // caller can tell input that stops early from input that is wrong. | |
| 921 | _note_if_at_end_of_input(location: LOCATION) is | |
| 922 | last_error_line = location.start_line | |
| 923 | ||
| 924 | if current_token == Lexical.TOKEN.END_OF_INPUT then | |
| 925 | end_of_input_error_locations.add(location) | |
| 926 | else | |
| 927 | errors_away_from_end_of_input = errors_away_from_end_of_input + 1 | |
| 928 | fi | |
| 929 | si | |
| 930 | si | |
| 931 | si |