OLD | NEW |
1 // Copyright 2009 The Go Authors. All rights reserved. | 1 // Copyright 2009 The Go Authors. All rights reserved. |
2 // Use of this source code is governed by a BSD-style | 2 // Use of this source code is governed by a BSD-style |
3 // license that can be found in the LICENSE file. | 3 // license that can be found in the LICENSE file. |
4 | 4 |
5 // Package scanner implements a scanner for Go source text. Takes a []byte as | 5 // Package scanner implements a scanner for Go source text. Takes a []byte as |
6 // source which can then be tokenized through repeated calls to the Scan | 6 // source which can then be tokenized through repeated calls to the Scan |
7 // function. Typical use: | 7 // function. Typical use: |
8 // | 8 // |
9 // var s Scanner | 9 // var s Scanner |
10 // fset := token.NewFileSet() // position information is relative to fset | 10 // fset := token.NewFileSet() // position information is relative to fset |
11 // file := fset.AddFile(filename, fset.Base(), len(src)) // register file | 11 // file := fset.AddFile(filename, fset.Base(), len(src)) // register file |
12 // s.Init(file, src, nil /* no error handler */, 0) | 12 // s.Init(file, src, nil /* no error handler */, 0) |
13 // for { | 13 // for { |
14 // pos, tok, lit := s.Scan() | 14 // pos, tok, lit := s.Scan() |
15 // if tok == token.EOF { | 15 // if tok == token.EOF { |
16 // break | 16 // break |
17 // } | 17 // } |
18 // // do something here with pos, tok, and lit | 18 // // do something here with pos, tok, and lit |
19 // } | 19 // } |
20 // | 20 // |
21 package scanner | 21 package scanner |
22 | 22 |
23 import ( | 23 import ( |
24 "bytes" | 24 "bytes" |
| 25 "fmt" |
25 "go/token" | 26 "go/token" |
26 "path/filepath" | 27 "path/filepath" |
27 "strconv" | 28 "strconv" |
28 "unicode" | 29 "unicode" |
29 "utf8" | 30 "utf8" |
30 ) | 31 ) |
31 | 32 |
32 | 33 |
33 // A Scanner holds the scanner's internal state while processing | 34 // A Scanner holds the scanner's internal state while processing |
34 // a given text. It can be allocated as part of another data | 35 // a given text. It can be allocated as part of another data |
(...skipping 632 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
667 if S.ch == '^' { | 668 if S.ch == '^' { |
668 S.next() | 669 S.next() |
669 tok = S.switch2(token.AND_NOT, token.AND_NOT_ASS
IGN) | 670 tok = S.switch2(token.AND_NOT, token.AND_NOT_ASS
IGN) |
670 } else { | 671 } else { |
671 tok = S.switch3(token.AND, token.AND_ASSIGN, '&'
, token.LAND) | 672 tok = S.switch3(token.AND, token.AND_ASSIGN, '&'
, token.LAND) |
672 } | 673 } |
673 case '|': | 674 case '|': |
674 tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LO
R) | 675 tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LO
R) |
675 default: | 676 default: |
676 if S.mode&AllowIllegalChars == 0 { | 677 if S.mode&AllowIllegalChars == 0 { |
677 » » » » S.error(offs, "illegal character "+strconv.Quote
Rune(ch)) | 678 » » » » S.error(offs, fmt.Sprintf("illegal character %#U
", ch)) |
678 } | 679 } |
679 insertSemi = S.insertSemi // preserve insertSemi info | 680 insertSemi = S.insertSemi // preserve insertSemi info |
680 } | 681 } |
681 } | 682 } |
682 | 683 |
683 if S.mode&InsertSemis != 0 { | 684 if S.mode&InsertSemis != 0 { |
684 S.insertSemi = insertSemi | 685 S.insertSemi = insertSemi |
685 } | 686 } |
686 | 687 |
687 // TODO(gri): The scanner API should change such that the literal string | 688 // TODO(gri): The scanner API should change such that the literal string |
688 // is only valid if an actual literal was scanned. This will | 689 // is only valid if an actual literal was scanned. This will |
689 // permit a more efficient implementation. | 690 // permit a more efficient implementation. |
690 return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) | 691 return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) |
691 } | 692 } |
OLD | NEW |