Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(317)

Side by Side Diff: src/pkg/go/scanner/scanner.go

Issue 4572049: code review 4572049: go/scanner: partially revert CL4538096 now that we have... (Closed)
Patch Set: diff -r 2fb655410b1a https://go.googlecode.com/hg/ Created 13 years, 10 months ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
« no previous file with comments | « no previous file | src/pkg/go/scanner/scanner_test.go » ('j') | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 // Copyright 2009 The Go Authors. All rights reserved. 1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style 2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file. 3 // license that can be found in the LICENSE file.
4 4
5 // Package scanner implements a scanner for Go source text. Takes a []byte as 5 // Package scanner implements a scanner for Go source text. Takes a []byte as
6 // source which can then be tokenized through repeated calls to the Scan 6 // source which can then be tokenized through repeated calls to the Scan
7 // function. Typical use: 7 // function. Typical use:
8 // 8 //
9 // var s Scanner 9 // var s Scanner
10 // fset := token.NewFileSet() // position information is relative to fset 10 // fset := token.NewFileSet() // position information is relative to fset
11 // file := fset.AddFile(filename, fset.Base(), len(src)) // register file 11 // file := fset.AddFile(filename, fset.Base(), len(src)) // register file
12 // s.Init(file, src, nil /* no error handler */, 0) 12 // s.Init(file, src, nil /* no error handler */, 0)
13 // for { 13 // for {
14 // pos, tok, lit := s.Scan() 14 // pos, tok, lit := s.Scan()
15 // if tok == token.EOF { 15 // if tok == token.EOF {
16 // break 16 // break
17 // } 17 // }
18 // // do something here with pos, tok, and lit 18 // // do something here with pos, tok, and lit
19 // } 19 // }
20 // 20 //
21 package scanner 21 package scanner
22 22
23 import ( 23 import (
24 "bytes" 24 "bytes"
25 "fmt"
25 "go/token" 26 "go/token"
26 "path/filepath" 27 "path/filepath"
27 "strconv" 28 "strconv"
28 "unicode" 29 "unicode"
29 "utf8" 30 "utf8"
30 ) 31 )
31 32
32 33
33 // A Scanner holds the scanner's internal state while processing 34 // A Scanner holds the scanner's internal state while processing
34 // a given text. It can be allocated as part of another data 35 // a given text. It can be allocated as part of another data
(...skipping 632 matching lines...) Expand 10 before | Expand all | Expand 10 after
667 if S.ch == '^' { 668 if S.ch == '^' {
668 S.next() 669 S.next()
669 tok = S.switch2(token.AND_NOT, token.AND_NOT_ASS IGN) 670 tok = S.switch2(token.AND_NOT, token.AND_NOT_ASS IGN)
670 } else { 671 } else {
671 tok = S.switch3(token.AND, token.AND_ASSIGN, '&' , token.LAND) 672 tok = S.switch3(token.AND, token.AND_ASSIGN, '&' , token.LAND)
672 } 673 }
673 case '|': 674 case '|':
674 tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LO R) 675 tok = S.switch3(token.OR, token.OR_ASSIGN, '|', token.LO R)
675 default: 676 default:
676 if S.mode&AllowIllegalChars == 0 { 677 if S.mode&AllowIllegalChars == 0 {
677 » » » » S.error(offs, "illegal character "+strconv.Quote Rune(ch)) 678 » » » » S.error(offs, fmt.Sprintf("illegal character %#U ", ch))
678 } 679 }
679 insertSemi = S.insertSemi // preserve insertSemi info 680 insertSemi = S.insertSemi // preserve insertSemi info
680 } 681 }
681 } 682 }
682 683
683 if S.mode&InsertSemis != 0 { 684 if S.mode&InsertSemis != 0 {
684 S.insertSemi = insertSemi 685 S.insertSemi = insertSemi
685 } 686 }
686 687
687 // TODO(gri): The scanner API should change such that the literal string 688 // TODO(gri): The scanner API should change such that the literal string
688 // is only valid if an actual literal was scanned. This will 689 // is only valid if an actual literal was scanned. This will
689 // permit a more efficient implementation. 690 // permit a more efficient implementation.
690 return S.file.Pos(offs), tok, string(S.src[offs:S.offset]) 691 return S.file.Pos(offs), tok, string(S.src[offs:S.offset])
691 } 692 }
OLDNEW
« no previous file with comments | « no previous file | src/pkg/go/scanner/scanner_test.go » ('j') | no next file with comments »

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b