Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(534)

Side by Side Diff: src/pkg/go/parser/parser.go

Issue 4291070: code review 4291070: go/scanner: return literal as string instead of []byte (Closed)
Patch Set: diff -r 4073ecdfc054 https://go.googlecode.com/hg/ Created 13 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2009 The Go Authors. All rights reserved. 1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style 2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file. 3 // license that can be found in the LICENSE file.
4 4
5 // A parser for Go source files. Input may be provided in a variety of 5 // A parser for Go source files. Input may be provided in a variety of
6 // forms (see the various Parse* functions); the output is an abstract 6 // forms (see the various Parse* functions); the output is an abstract
7 // syntax tree (AST) representing the Go source. The parser is invoked 7 // syntax tree (AST) representing the Go source. The parser is invoked
8 // through one of the Parse* functions. 8 // through one of the Parse* functions.
9 // 9 //
10 package parser 10 package parser
(...skipping 29 matching lines...) Expand all
40 mode uint // parsing mode 40 mode uint // parsing mode
41 trace bool // == (mode & Trace != 0) 41 trace bool // == (mode & Trace != 0)
42 indent uint // indentation used for tracing output 42 indent uint // indentation used for tracing output
43 43
44 // Comments 44 // Comments
45 comments []*ast.CommentGroup 45 comments []*ast.CommentGroup
46 leadComment *ast.CommentGroup // last lead comment 46 leadComment *ast.CommentGroup // last lead comment
47 lineComment *ast.CommentGroup // last line comment 47 lineComment *ast.CommentGroup // last line comment
48 48
49 // Next token 49 // Next token
50 » pos token.Pos // token position 50 » pos token.Pos // token position
51 » tok token.Token // one token look-ahead 51 » tok token.Token // one token look-ahead
52 » lit_ []byte // token literal (slice into original source, don't hol d on to it) 52 » lit string // token literal
53 53
54 // Non-syntactic parser control 54 // Non-syntactic parser control
55 exprLev int // < 0: in control clause, >= 0: in expression 55 exprLev int // < 0: in control clause, >= 0: in expression
56 56
57 // Ordinary identifer scopes 57 // Ordinary identifer scopes
58 pkgScope *ast.Scope // pkgScope.Outer == nil 58 pkgScope *ast.Scope // pkgScope.Outer == nil
59 topScope *ast.Scope // top-most scope; may be pkgScope 59 topScope *ast.Scope // top-most scope; may be pkgScope
60 unresolved []*ast.Ident // unresolved identifiers 60 unresolved []*ast.Ident // unresolved identifiers
61 imports []*ast.ImportSpec // list of imports 61 imports []*ast.ImportSpec // list of imports
62 62
(...skipping 26 matching lines...) Expand all
89 // set up the pkgScope here (as opposed to in parseFile) because 89 // set up the pkgScope here (as opposed to in parseFile) because
90 // there are other parser entry points (ParseExpr, etc.) 90 // there are other parser entry points (ParseExpr, etc.)
91 p.openScope() 91 p.openScope()
92 p.pkgScope = p.topScope 92 p.pkgScope = p.topScope
93 93
94 // for the same reason, set up a label scope 94 // for the same reason, set up a label scope
95 p.openLabelScope() 95 p.openLabelScope()
96 } 96 }
97 97
98 98
99 func (p *parser) lit() []byte {
100 // make a copy of p.lit_ so that we don't hold on to
101 // a copy of the entire source indirectly in the AST
102 t := make([]byte, len(p.lit_))
103 copy(t, p.lit_)
104 return t
105 }
106
107
108 // ---------------------------------------------------------------------------- 99 // ----------------------------------------------------------------------------
109 // Scoping support 100 // Scoping support
110 101
111 func (p *parser) openScope() { 102 func (p *parser) openScope() {
112 p.topScope = ast.NewScope(p.topScope) 103 p.topScope = ast.NewScope(p.topScope)
113 } 104 }
114 105
115 106
116 func (p *parser) closeScope() { 107 func (p *parser) closeScope() {
117 p.topScope = p.topScope.Outer 108 p.topScope = p.topScope.Outer
(...skipping 136 matching lines...) Expand 10 before | Expand all | Expand 10 after
254 // Advance to the next token. 245 // Advance to the next token.
255 func (p *parser) next0() { 246 func (p *parser) next0() {
256 // Because of one-token look-ahead, print the previous token 247 // Because of one-token look-ahead, print the previous token
257 // when tracing as it provides a more readable output. The 248 // when tracing as it provides a more readable output. The
258 // very first token (!p.pos.IsValid()) is not initialized 249 // very first token (!p.pos.IsValid()) is not initialized
259 // (it is token.ILLEGAL), so don't print it . 250 // (it is token.ILLEGAL), so don't print it .
260 if p.trace && p.pos.IsValid() { 251 if p.trace && p.pos.IsValid() {
261 s := p.tok.String() 252 s := p.tok.String()
262 switch { 253 switch {
263 case p.tok.IsLiteral(): 254 case p.tok.IsLiteral():
264 » » » p.printTrace(s, string(p.lit_)) 255 » » » p.printTrace(s, p.lit)
265 case p.tok.IsOperator(), p.tok.IsKeyword(): 256 case p.tok.IsOperator(), p.tok.IsKeyword():
266 p.printTrace("\"" + s + "\"") 257 p.printTrace("\"" + s + "\"")
267 default: 258 default:
268 p.printTrace(s) 259 p.printTrace(s)
269 } 260 }
270 } 261 }
271 262
272 » p.pos, p.tok, p.lit_ = p.scanner.Scan() 263 » p.pos, p.tok, p.lit = p.scanner.Scan()
273 } 264 }
274 265
275 // Consume a comment and return it and the line on which it ends. 266 // Consume a comment and return it and the line on which it ends.
276 func (p *parser) consumeComment() (comment *ast.Comment, endline int) { 267 func (p *parser) consumeComment() (comment *ast.Comment, endline int) {
277 // /*-style comments may end on a different line than where they start. 268 // /*-style comments may end on a different line than where they start.
278 // Scan the comment for '\n' chars and adjust endline accordingly. 269 // Scan the comment for '\n' chars and adjust endline accordingly.
279 endline = p.file.Line(p.pos) 270 endline = p.file.Line(p.pos)
280 » if p.lit_[1] == '*' { 271 » if p.lit[1] == '*' {
281 » » for _, b := range p.lit_ { 272 » » // don't use range here - no need to decode Unicode code points
282 » » » if b == '\n' { 273 » » for i := 0; i < len(p.lit); i++ {
274 » » » if p.lit[i] == '\n' {
283 endline++ 275 endline++
284 } 276 }
285 } 277 }
286 } 278 }
287 279
288 » comment = &ast.Comment{p.pos, p.lit()} 280 » comment = &ast.Comment{p.pos, p.lit}
289 p.next0() 281 p.next0()
290 282
291 return 283 return
292 } 284 }
293 285
294 286
295 // Consume a group of adjacent comments, add it to the parser's 287 // Consume a group of adjacent comments, add it to the parser's
296 // comments list, and return it together with the line at which 288 // comments list, and return it together with the line at which
297 // the last comment in the group ends. An empty line or non-comment 289 // the last comment in the group ends. An empty line or non-comment
298 // token terminates a comment group. 290 // token terminates a comment group.
(...skipping 69 matching lines...) Expand 10 before | Expand all | Expand 10 after
368 func (p *parser) error(pos token.Pos, msg string) { 360 func (p *parser) error(pos token.Pos, msg string) {
369 p.Error(p.file.Position(pos), msg) 361 p.Error(p.file.Position(pos), msg)
370 } 362 }
371 363
372 364
373 func (p *parser) errorExpected(pos token.Pos, msg string) { 365 func (p *parser) errorExpected(pos token.Pos, msg string) {
374 msg = "expected " + msg 366 msg = "expected " + msg
375 if pos == p.pos { 367 if pos == p.pos {
376 // the error happened at the current position; 368 // the error happened at the current position;
377 // make the error message more specific 369 // make the error message more specific
378 » » if p.tok == token.SEMICOLON && p.lit_[0] == '\n' { 370 » » if p.tok == token.SEMICOLON && p.lit[0] == '\n' {
379 msg += ", found newline" 371 msg += ", found newline"
380 } else { 372 } else {
381 msg += ", found '" + p.tok.String() + "'" 373 msg += ", found '" + p.tok.String() + "'"
382 if p.tok.IsLiteral() { 374 if p.tok.IsLiteral() {
383 » » » » msg += " " + string(p.lit_) 375 » » » » msg += " " + p.lit
384 } 376 }
385 } 377 }
386 } 378 }
387 p.error(pos, msg) 379 p.error(pos, msg)
388 } 380 }
389 381
390 382
391 func (p *parser) expect(tok token.Token) token.Pos { 383 func (p *parser) expect(tok token.Token) token.Pos {
392 pos := p.pos 384 pos := p.pos
393 if p.tok != tok { 385 if p.tok != tok {
(...skipping 18 matching lines...) Expand all
412 } 404 }
413 405
414 406
415 // ---------------------------------------------------------------------------- 407 // ----------------------------------------------------------------------------
416 // Identifiers 408 // Identifiers
417 409
418 func (p *parser) parseIdent() *ast.Ident { 410 func (p *parser) parseIdent() *ast.Ident {
419 pos := p.pos 411 pos := p.pos
420 name := "_" 412 name := "_"
421 if p.tok == token.IDENT { 413 if p.tok == token.IDENT {
422 » » name = string(p.lit_) 414 » » name = p.lit
423 p.next() 415 p.next()
424 } else { 416 } else {
425 p.expect(token.IDENT) // use expect() error handling 417 p.expect(token.IDENT) // use expect() error handling
426 } 418 }
427 return &ast.Ident{pos, name, nil} 419 return &ast.Ident{pos, name, nil}
428 } 420 }
429 421
430 422
431 func (p *parser) parseIdentList() (list []*ast.Ident) { 423 func (p *parser) parseIdentList() (list []*ast.Ident) {
432 if p.trace { 424 if p.trace {
(...skipping 141 matching lines...) Expand 10 before | Expand all | Expand 10 after
574 } 566 }
575 567
576 doc := p.leadComment 568 doc := p.leadComment
577 569
578 // fields 570 // fields
579 list, typ := p.parseVarList(false) 571 list, typ := p.parseVarList(false)
580 572
581 // optional tag 573 // optional tag
582 var tag *ast.BasicLit 574 var tag *ast.BasicLit
583 if p.tok == token.STRING { 575 if p.tok == token.STRING {
584 » » tag = &ast.BasicLit{p.pos, p.tok, p.lit()} 576 » » tag = &ast.BasicLit{p.pos, p.tok, p.lit}
585 p.next() 577 p.next()
586 } 578 }
587 579
588 // analyze case 580 // analyze case
589 var idents []*ast.Ident 581 var idents []*ast.Ident
590 if typ != nil { 582 if typ != nil {
591 // IdentifierList Type 583 // IdentifierList Type
592 idents = p.makeIdentList(list) 584 idents = p.makeIdentList(list)
593 } else { 585 } else {
594 // ["*"] TypeName (AnonymousField) 586 // ["*"] TypeName (AnonymousField)
(...skipping 422 matching lines...) Expand 10 before | Expand all | Expand 10 after
1017 1009
1018 switch p.tok { 1010 switch p.tok {
1019 case token.IDENT: 1011 case token.IDENT:
1020 x := p.parseIdent() 1012 x := p.parseIdent()
1021 if !lhs { 1013 if !lhs {
1022 p.resolve(x) 1014 p.resolve(x)
1023 } 1015 }
1024 return x 1016 return x
1025 1017
1026 case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING: 1018 case token.INT, token.FLOAT, token.IMAG, token.CHAR, token.STRING:
1027 » » x := &ast.BasicLit{p.pos, p.tok, p.lit()} 1019 » » x := &ast.BasicLit{p.pos, p.tok, p.lit}
1028 p.next() 1020 p.next()
1029 return x 1021 return x
1030 1022
1031 case token.LPAREN: 1023 case token.LPAREN:
1032 lparen := p.pos 1024 lparen := p.pos
1033 p.next() 1025 p.next()
1034 p.exprLev++ 1026 p.exprLev++
1035 x := p.parseRhs() 1027 x := p.parseRhs()
1036 p.exprLev-- 1028 p.exprLev--
1037 rparen := p.expect(token.RPAREN) 1029 rparen := p.expect(token.RPAREN)
(...skipping 933 matching lines...) Expand 10 before | Expand all | Expand 10 after
1971 switch p.tok { 1963 switch p.tok {
1972 case token.PERIOD: 1964 case token.PERIOD:
1973 ident = &ast.Ident{p.pos, ".", nil} 1965 ident = &ast.Ident{p.pos, ".", nil}
1974 p.next() 1966 p.next()
1975 case token.IDENT: 1967 case token.IDENT:
1976 ident = p.parseIdent() 1968 ident = p.parseIdent()
1977 } 1969 }
1978 1970
1979 var path *ast.BasicLit 1971 var path *ast.BasicLit
1980 if p.tok == token.STRING { 1972 if p.tok == token.STRING {
1981 » » path = &ast.BasicLit{p.pos, p.tok, p.lit()} 1973 » » path = &ast.BasicLit{p.pos, p.tok, p.lit}
1982 p.next() 1974 p.next()
1983 } else { 1975 } else {
1984 p.expect(token.STRING) // use expect() error handling 1976 p.expect(token.STRING) // use expect() error handling
1985 } 1977 }
1986 p.expectSemi() // call before accessing p.linecomment 1978 p.expectSemi() // call before accessing p.linecomment
1987 1979
1988 // collect imports 1980 // collect imports
1989 spec := &ast.ImportSpec{doc, ident, path, p.lineComment} 1981 spec := &ast.ImportSpec{doc, ident, path, p.lineComment}
1990 p.imports = append(p.imports, spec) 1982 p.imports = append(p.imports, spec)
1991 1983
(...skipping 254 matching lines...) Expand 10 before | Expand all | Expand 10 after
2246 ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unreso lved sentinel 2238 ident.Obj = p.pkgScope.Lookup(ident.Name) // also removes unreso lved sentinel
2247 if ident.Obj == nil { 2239 if ident.Obj == nil {
2248 p.unresolved[i] = ident 2240 p.unresolved[i] = ident
2249 i++ 2241 i++
2250 } 2242 }
2251 } 2243 }
2252 2244
2253 // TODO(gri): store p.imports in AST 2245 // TODO(gri): store p.imports in AST
2254 return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unreso lved[0:i], p.comments} 2246 return &ast.File{doc, pos, ident, decls, p.pkgScope, p.imports, p.unreso lved[0:i], p.comments}
2255 } 2247 }
OLDNEW

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b