Left: | ||
Right: |
LEFT | RIGHT |
---|---|
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // A scanner for Go source text. Takes a []byte as source which can | 5 // A scanner for Go source text. Takes a []byte as source which can |
6 // then be tokenized through repeated calls to the Scan function. | 6 // then be tokenized through repeated calls to the Scan function. |
7 // Typical use: | 7 // Typical use: |
8 // | 8 // |
9 // var s Scanner | 9 // var s Scanner |
10 // fset := token.NewFileSet() // position information is relative to fset | 10 // fset := token.NewFileSet() // position information is relative to fset |
(...skipping 520 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
531 if S.ch == '=' { | 531 if S.ch == '=' { |
532 S.next() | 532 S.next() |
533 return tok3 | 533 return tok3 |
534 } | 534 } |
535 return tok2 | 535 return tok2 |
536 } | 536 } |
537 return tok0 | 537 return tok0 |
538 } | 538 } |
539 | 539 |
540 | 540 |
541 // Scan scans the next token and returns the token position pos, | 541 // Scan scans the next token and returns the token position, |
542 // the token tok, and the literal text lit corresponding to the | 542 // the token, and the literal string corresponding to the |
543 // token. The source end is indicated by token.EOF. | 543 // token. The source end is indicated by token.EOF. |
544 // | 544 // |
545 // If the returned token is token.SEMICOLON, the corresponding | 545 // If the returned token is token.SEMICOLON, the corresponding |
546 // literal value is ";" if the semicolon was present in the source, | 546 // literal string is ";" if the semicolon was present in the source, |
547 // and "\n" if the semicolon was inserted because of a newline or | 547 // and "\n" if the semicolon was inserted because of a newline or |
548 // at EOF. | 548 // at EOF. |
549 // | 549 // |
550 // For more tolerant parsing, Scan will return a valid token if | 550 // For more tolerant parsing, Scan will return a valid token if |
551 // possible even if a syntax error was encountered. Thus, even | 551 // possible even if a syntax error was encountered. Thus, even |
552 // if the resulting token sequence contains no illegal tokens, | 552 // if the resulting token sequence contains no illegal tokens, |
553 // a client may not assume that no error occurred. Instead it | 553 // a client may not assume that no error occurred. Instead it |
554 // must check the scanner's ErrorCount or the number of calls | 554 // must check the scanner's ErrorCount or the number of calls |
555 // of the error handler, if there was one installed. | 555 // of the error handler, if there was one installed. |
556 // | 556 // |
(...skipping 145 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
702 if S.mode&AllowIllegalChars == 0 { | 702 if S.mode&AllowIllegalChars == 0 { |
703 S.error(offs, "illegal character "+charString(ch )) | 703 S.error(offs, "illegal character "+charString(ch )) |
704 } | 704 } |
705 insertSemi = S.insertSemi // preserve insertSemi info | 705 insertSemi = S.insertSemi // preserve insertSemi info |
706 } | 706 } |
707 } | 707 } |
708 | 708 |
709 if S.mode&InsertSemis != 0 { | 709 if S.mode&InsertSemis != 0 { |
710 S.insertSemi = insertSemi | 710 S.insertSemi = insertSemi |
711 } | 711 } |
712 | |
713 // TODO(gri): The scanner API should change such that the literal string | |
rsc
2011/03/29 03:52:20
I think the more efficient implementation can be d
gri
2011/03/29 04:44:34
+ taking care of AllowIllegalChars (albeit now tha
| |
714 // is only valid if an actual literal was scanned. This will | |
715 // permit a more efficient implementation. | |
712 return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) | 716 return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) |
713 } | 717 } |
LEFT | RIGHT |