Rietveld Code Review Tool
Help | Bug tracker | Discussion group | Source code | Sign in
(1136)

Delta Between Two Patch Sets: src/pkg/exp/datafmt/parser.go

Issue 4291070: code review 4291070: go/scanner: return literal as string instead of []byte (Closed)
Left Patch Set: diff -r 56b847e640f3 https://go.googlecode.com/hg/ Created 13 years ago
Right Patch Set: diff -r 4073ecdfc054 https://go.googlecode.com/hg/ Created 13 years ago
Left:
Right:
Use n/p to move between diff chunks; N/P to move between comments. Please Sign in to add in-line comments.
Jump to:
Left: Side by side diff | Download
Right: Side by side diff | Download
LEFTRIGHT
1 ÿ// Copyright 2009 The Go Authors. All rights reserved.ÿ 1 // Copyright 2009 The Go Authors. All rights reserved.
2 ÿ// Use of this source code is governed by a BSD-styleÿ 2 // Use of this source code is governed by a BSD-style
3 ÿ// license that can be found in the LICENSE file.ÿ 3 // license that can be found in the LICENSE file.
4 4
5 package datafmt 5 package datafmt
6 6
7 import ( 7 import (
8 "container/vector" 8 "container/vector"
9 "go/scanner" 9 "go/scanner"
10 "go/token" 10 "go/token"
11 "os" 11 "os"
12 "strconv" 12 "strconv"
13 "strings" 13 "strings"
14 ) 14 )
15 15
16 ÿ// ---------------------------------------------------------------------------- ÿ 16 // ----------------------------------------------------------------------------
17 ÿ// Parsingÿ 17 // Parsing
18 18
19 type parser struct { 19 type parser struct {
20 scanner.ErrorVector 20 scanner.ErrorVector
21 scanner scanner.Scanner 21 scanner scanner.Scanner
22 file *token.File 22 file *token.File
23 » pos token.Pos ÿ// token positionÿ 23 » pos token.Pos // token position
24 » tok token.Token ÿ// one token look-aheadÿ 24 » tok token.Token // one token look-ahead
25 » lit string ÿ// token literalÿ 25 » lit string // token literal
26 26
27 » packs map[string]string ÿ// PackageName -> ImportPathÿ 27 » packs map[string]string // PackageName -> ImportPath
28 » rules map[string]expr ÿ// RuleName -> Expressionÿ 28 » rules map[string]expr // RuleName -> Expression
29 } 29 }
30 30
31 31
32 func (p *parser) next() { 32 func (p *parser) next() {
33 p.pos, p.tok, p.lit = p.scanner.Scan() 33 p.pos, p.tok, p.lit = p.scanner.Scan()
34 switch p.tok { 34 switch p.tok {
35 case token.CHAN, token.FUNC, token.INTERFACE, token.MAP, token.STRUCT: 35 case token.CHAN, token.FUNC, token.INTERFACE, token.MAP, token.STRUCT:
36 » » ÿ// Go keywords for composite types are type namesÿ 36 » » // Go keywords for composite types are type names
37 » » ÿ// returned by reflect. Accept them as identifiers.ÿ 37 » » // returned by reflect. Accept them as identifiers.
38 » » p.tok = token.IDENT ÿ// p.lit is already set correctlyÿ 38 » » p.tok = token.IDENT // p.lit is already set correctly
39 } 39 }
40 } 40 }
41 41
42 42
43 func (p *parser) init(fset *token.FileSet, filename string, src []byte) { 43 func (p *parser) init(fset *token.FileSet, filename string, src []byte) {
44 p.ErrorVector.Reset() 44 p.ErrorVector.Reset()
45 p.file = fset.AddFile(filename, fset.Base(), len(src)) 45 p.file = fset.AddFile(filename, fset.Base(), len(src))
46 » p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) ÿ// return '@' as token.ILLEGAL w/o error messageÿ 46 » p.scanner.Init(p.file, src, p, scanner.AllowIllegalChars) // return '@' as token.ILLEGAL w/o error message
47 » p.next() ÿ// initialize s pos, tok, litÿ 47 » p.next() // initializes pos, tok, lit
48 p.packs = make(map[string]string) 48 p.packs = make(map[string]string)
49 p.rules = make(map[string]expr) 49 p.rules = make(map[string]expr)
50 } 50 }
51 51
52 52
53 func (p *parser) error(pos token.Pos, msg string) { 53 func (p *parser) error(pos token.Pos, msg string) {
54 p.Error(p.file.Position(pos), msg) 54 p.Error(p.file.Position(pos), msg)
55 } 55 }
56 56
57 57
58 func (p *parser) errorExpected(pos token.Pos, msg string) { 58 func (p *parser) errorExpected(pos token.Pos, msg string) {
59 msg = "expected " + msg 59 msg = "expected " + msg
60 if pos == p.pos { 60 if pos == p.pos {
61 » » ÿ// the error happened at the current position;ÿ 61 » » // the error happened at the current position;
62 » » ÿ// make the error message more specificÿ 62 » » // make the error message more specific
63 msg += ", found '" + p.tok.String() + "'" 63 msg += ", found '" + p.tok.String() + "'"
64 if p.tok.IsLiteral() { 64 if p.tok.IsLiteral() {
65 msg += " " + p.lit 65 msg += " " + p.lit
66 } 66 }
67 } 67 }
68 p.error(pos, msg) 68 p.error(pos, msg)
69 } 69 }
70 70
71 71
72 func (p *parser) expect(tok token.Token) token.Pos { 72 func (p *parser) expect(tok token.Token) token.Pos {
73 pos := p.pos 73 pos := p.pos
74 if p.tok != tok { 74 if p.tok != tok {
75 p.errorExpected(pos, "'"+tok.String()+"'") 75 p.errorExpected(pos, "'"+tok.String()+"'")
76 } 76 }
77 » p.next() ÿ// make progress in any caseÿ 77 » p.next() // make progress in any case
78 return pos 78 return pos
79 } 79 }
80 80
81 81
82 func (p *parser) parseIdentifier() string { 82 func (p *parser) parseIdentifier() string {
83 name := p.lit 83 name := p.lit
84 p.expect(token.IDENT) 84 p.expect(token.IDENT)
85 return name 85 return name
86 } 86 }
87 87
88 88
89 func (p *parser) parseTypeName() (string, bool) { 89 func (p *parser) parseTypeName() (string, bool) {
90 pos := p.pos 90 pos := p.pos
91 name, isIdent := p.parseIdentifier(), true 91 name, isIdent := p.parseIdentifier(), true
92 if p.tok == token.PERIOD { 92 if p.tok == token.PERIOD {
93 » » ÿ// got a package name, lookup packageÿ 93 » » // got a package name, lookup package
94 if importPath, found := p.packs[name]; found { 94 if importPath, found := p.packs[name]; found {
95 name = importPath 95 name = importPath
96 } else { 96 } else {
97 p.error(pos, "package not declared: "+name) 97 p.error(pos, "package not declared: "+name)
98 } 98 }
99 p.next() 99 p.next()
100 name, isIdent = name+"."+p.parseIdentifier(), false 100 name, isIdent = name+"."+p.parseIdentifier(), false
101 } 101 }
102 return name, isIdent 102 return name, isIdent
103 } 103 }
104 104
105 105
106 ÿ// Parses a rule name and returns it. If the rule name isÿ 106 // Parses a rule name and returns it. If the rule name is
107 ÿ// a package-qualified type name, the package name is resolved.ÿ 107 // a package-qualified type name, the package name is resolved.
108 ÿ// The 2nd result value is true iff the rule name consists of aÿ 108 // The 2nd result value is true iff the rule name consists of a
109 ÿ// single identifier only (and thus could be a package name).ÿ 109 // single identifier only (and thus could be a package name).
110 ÿ//ÿ 110 //
111 func (p *parser) parseRuleName() (string, bool) { 111 func (p *parser) parseRuleName() (string, bool) {
112 name, isIdent := "", false 112 name, isIdent := "", false
113 switch p.tok { 113 switch p.tok {
114 case token.IDENT: 114 case token.IDENT:
115 name, isIdent = p.parseTypeName() 115 name, isIdent = p.parseTypeName()
116 case token.DEFAULT: 116 case token.DEFAULT:
117 name = "default" 117 name = "default"
118 p.next() 118 p.next()
119 case token.QUO: 119 case token.QUO:
120 name = "/" 120 name = "/"
121 p.next() 121 p.next()
122 default: 122 default:
123 p.errorExpected(p.pos, "rule name") 123 p.errorExpected(p.pos, "rule name")
124 » » p.next() ÿ// make progress in any caseÿ 124 » » p.next() // make progress in any case
125 } 125 }
126 return name, isIdent 126 return name, isIdent
127 } 127 }
128 128
129 129
130 func (p *parser) parseString() string { 130 func (p *parser) parseString() string {
131 s := "" 131 s := ""
132 if p.tok == token.STRING { 132 if p.tok == token.STRING {
133 s, _ = strconv.Unquote(p.lit) 133 s, _ = strconv.Unquote(p.lit)
134 » » ÿ// Unquote may fail with an error, but only if the scanner foun dÿ 134 » » // Unquote may fail with an error, but only if the scanner found
135 » » ÿ// an illegal string in the first place. In this case the error ÿ 135 » » // an illegal string in the first place. In this case the error
136 » » ÿ// has already been reported.ÿ 136 » » // has already been reported.
137 p.next() 137 p.next()
138 return s 138 return s
139 } else { 139 } else {
140 p.expect(token.STRING) 140 p.expect(token.STRING)
141 } 141 }
142 return s 142 return s
143 } 143 }
144 144
145 145
146 func (p *parser) parseLiteral() literal { 146 func (p *parser) parseLiteral() literal {
147 s := []byte(p.parseString()) 147 s := []byte(p.parseString())
148 148
149 » ÿ// A string literal may contain %-format specifiers. To simplifyÿ 149 » // A string literal may contain %-format specifiers. To simplify
150 » ÿ// and speed up printing of the literal, split it into segmentsÿ 150 » // and speed up printing of the literal, split it into segments
151 » ÿ// that start with "%" possibly followed by a last segment thatÿ 151 » // that start with "%" possibly followed by a last segment that
152 » ÿ// starts with some other character.ÿ 152 » // starts with some other character.
153 var list vector.Vector 153 var list vector.Vector
154 i0 := 0 154 i0 := 0
155 for i := 0; i < len(s); i++ { 155 for i := 0; i < len(s); i++ {
156 if s[i] == '%' && i+1 < len(s) { 156 if s[i] == '%' && i+1 < len(s) {
157 » » » ÿ// the next segment starts with a % formatÿ 157 » » » // the next segment starts with a % format
158 if i0 < i { 158 if i0 < i {
159 » » » » ÿ// the current segment is not empty, split it o ffÿ 159 » » » » // the current segment is not empty, split it of f
160 list.Push(s[i0:i]) 160 list.Push(s[i0:i])
161 i0 = i 161 i0 = i
162 } 162 }
163 » » » i++ ÿ// skip %; let loop skip over char after %ÿ 163 » » » i++ // skip %; let loop skip over char after %
164 » » } 164 » » }
165 » } 165 » }
166 » ÿ// the final segment may start with any characterÿ 166 » // the final segment may start with any character
167 » ÿ// (it is empty iff the string is empty)ÿ 167 » // (it is empty iff the string is empty)
168 list.Push(s[i0:]) 168 list.Push(s[i0:])
169 169
170 » ÿ// convert list into a literalÿ 170 » // convert list into a literal
171 lit := make(literal, list.Len()) 171 lit := make(literal, list.Len())
172 for i := 0; i < list.Len(); i++ { 172 for i := 0; i < list.Len(); i++ {
173 lit[i] = list.At(i).([]byte) 173 lit[i] = list.At(i).([]byte)
174 } 174 }
175 175
176 return lit 176 return lit
177 } 177 }
178 178
179 179
180 func (p *parser) parseField() expr { 180 func (p *parser) parseField() expr {
(...skipping 48 matching lines...) Expand 10 before | Expand all | Expand 10 after
229 x = p.parseExpression() 229 x = p.parseExpression()
230 var div expr 230 var div expr
231 if p.tok == token.QUO { 231 if p.tok == token.QUO {
232 p.next() 232 p.next()
233 div = p.parseExpression() 233 div = p.parseExpression()
234 } 234 }
235 x = &repetition{x, div} 235 x = &repetition{x, div}
236 p.expect(token.RBRACE) 236 p.expect(token.RBRACE)
237 237
238 default: 238 default:
239 » » x = p.parseField() ÿ// may be nilÿ 239 » » x = p.parseField() // may be nil
240 } 240 }
241 241
242 return x 242 return x
243 } 243 }
244 244
245 245
246 func (p *parser) parseSequence() expr { 246 func (p *parser) parseSequence() expr {
247 var list vector.Vector 247 var list vector.Vector
248 248
249 for x := p.parseOperand(); x != nil; x = p.parseOperand() { 249 for x := p.parseOperand(); x != nil; x = p.parseOperand() {
250 list.Push(x) 250 list.Push(x)
251 } 251 }
252 252
253 » ÿ// no need for a sequence if list.Len() < 2ÿ 253 » // no need for a sequence if list.Len() < 2
254 switch list.Len() { 254 switch list.Len() {
255 case 0: 255 case 0:
256 return nil 256 return nil
257 case 1: 257 case 1:
258 return list.At(0).(expr) 258 return list.At(0).(expr)
259 } 259 }
260 260
261 » ÿ// convert list into a sequenceÿ 261 » // convert list into a sequence
262 seq := make(sequence, list.Len()) 262 seq := make(sequence, list.Len())
263 for i := 0; i < list.Len(); i++ { 263 for i := 0; i < list.Len(); i++ {
264 seq[i] = list.At(i).(expr) 264 seq[i] = list.At(i).(expr)
265 } 265 }
266 return seq 266 return seq
267 } 267 }
268 268
269 269
270 func (p *parser) parseExpression() expr { 270 func (p *parser) parseExpression() expr {
271 var list vector.Vector 271 var list vector.Vector
272 272
273 for { 273 for {
274 x := p.parseSequence() 274 x := p.parseSequence()
275 if x != nil { 275 if x != nil {
276 list.Push(x) 276 list.Push(x)
277 } 277 }
278 if p.tok != token.OR { 278 if p.tok != token.OR {
279 break 279 break
280 } 280 }
281 p.next() 281 p.next()
282 } 282 }
283 283
284 » ÿ// no need for an alternatives if list.Len() < 2ÿ 284 » // no need for an alternatives if list.Len() < 2
285 switch list.Len() { 285 switch list.Len() {
286 case 0: 286 case 0:
287 return nil 287 return nil
288 case 1: 288 case 1:
289 return list.At(0).(expr) 289 return list.At(0).(expr)
290 } 290 }
291 291
292 » ÿ// convert list into a alternativesÿ 292 » // convert list into a alternatives
293 alt := make(alternatives, list.Len()) 293 alt := make(alternatives, list.Len())
294 for i := 0; i < list.Len(); i++ { 294 for i := 0; i < list.Len(); i++ {
295 alt[i] = list.At(i).(expr) 295 alt[i] = list.At(i).(expr)
296 } 296 }
297 return alt 297 return alt
298 } 298 }
299 299
300 300
301 func (p *parser) parseFormat() { 301 func (p *parser) parseFormat() {
302 for p.tok != token.EOF { 302 for p.tok != token.EOF {
303 pos := p.pos 303 pos := p.pos
304 304
305 name, isIdent := p.parseRuleName() 305 name, isIdent := p.parseRuleName()
306 switch p.tok { 306 switch p.tok {
307 case token.STRING: 307 case token.STRING:
308 » » » ÿ// package declarationÿ 308 » » » // package declaration
309 importPath := p.parseString() 309 importPath := p.parseString()
310 310
311 » » » ÿ// add package declarationÿ 311 » » » // add package declaration
312 if !isIdent { 312 if !isIdent {
313 p.error(pos, "illegal package name: "+name) 313 p.error(pos, "illegal package name: "+name)
314 } else if _, found := p.packs[name]; !found { 314 } else if _, found := p.packs[name]; !found {
315 p.packs[name] = importPath 315 p.packs[name] = importPath
316 } else { 316 } else {
317 p.error(pos, "package already declared: "+name) 317 p.error(pos, "package already declared: "+name)
318 } 318 }
319 319
320 case token.ASSIGN: 320 case token.ASSIGN:
321 » » » ÿ// format ruleÿ 321 » » » // format rule
322 p.next() 322 p.next()
323 x := p.parseExpression() 323 x := p.parseExpression()
324 324
325 » » » ÿ// add ruleÿ 325 » » » // add rule
326 if _, found := p.rules[name]; !found { 326 if _, found := p.rules[name]; !found {
327 p.rules[name] = x 327 p.rules[name] = x
328 } else { 328 } else {
329 p.error(pos, "format rule already declared: "+na me) 329 p.error(pos, "format rule already declared: "+na me)
330 } 330 }
331 331
332 default: 332 default:
333 p.errorExpected(p.pos, "package declaration or format ru le") 333 p.errorExpected(p.pos, "package declaration or format ru le")
334 » » » p.next() ÿ// make progress in any caseÿ 334 » » » p.next() // make progress in any case
335 } 335 }
336 336
337 if p.tok == token.SEMICOLON { 337 if p.tok == token.SEMICOLON {
338 p.next() 338 p.next()
339 } else { 339 } else {
340 break 340 break
341 } 341 }
342 } 342 }
343 p.expect(token.EOF) 343 p.expect(token.EOF)
344 } 344 }
345 345
346 346
347 func remap(p *parser, name string) string { 347 func remap(p *parser, name string) string {
348 i := strings.Index(name, ".") 348 i := strings.Index(name, ".")
349 if i >= 0 { 349 if i >= 0 {
350 packageName, suffix := name[0:i], name[i:] 350 packageName, suffix := name[0:i], name[i:]
351 » » ÿ// lookup packageÿ 351 » » // lookup package
352 if importPath, found := p.packs[packageName]; found { 352 if importPath, found := p.packs[packageName]; found {
353 name = importPath + suffix 353 name = importPath + suffix
354 } else { 354 } else {
355 var invalidPos token.Position 355 var invalidPos token.Position
356 p.Error(invalidPos, "package not declared: "+packageName ) 356 p.Error(invalidPos, "package not declared: "+packageName )
357 } 357 }
358 } 358 }
359 return name 359 return name
360 } 360 }
361 361
362 362
363 ÿ// Parse parses a set of format productions from source src. Customÿ 363 // Parse parses a set of format productions from source src. Custom
364 ÿ// formatters may be provided via a map of formatter functions. Ifÿ 364 // formatters may be provided via a map of formatter functions. If
365 ÿ// there are no errors, the result is a Format and the error is nil.ÿ 365 // there are no errors, the result is a Format and the error is nil.
366 ÿ// Otherwise the format is nil and a non-empty ErrorList is returned.ÿ 366 // Otherwise the format is nil and a non-empty ErrorList is returned.
367 ÿ//ÿ 367 //
368 func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) { 368 func Parse(fset *token.FileSet, filename string, src []byte, fmap FormatterMap) (Format, os.Error) {
369 » ÿ// parse sourceÿ 369 » // parse source
370 var p parser 370 var p parser
371 p.init(fset, filename, src) 371 p.init(fset, filename, src)
372 p.parseFormat() 372 p.parseFormat()
373 373
374 » ÿ// add custom formatters, if anyÿ 374 » // add custom formatters, if any
375 for name, form := range fmap { 375 for name, form := range fmap {
376 name = remap(&p, name) 376 name = remap(&p, name)
377 if _, found := p.rules[name]; !found { 377 if _, found := p.rules[name]; !found {
378 p.rules[name] = &custom{name, form} 378 p.rules[name] = &custom{name, form}
379 } else { 379 } else {
380 var invalidPos token.Position 380 var invalidPos token.Position
381 p.Error(invalidPos, "formatter already declared: "+name) 381 p.Error(invalidPos, "formatter already declared: "+name)
382 } 382 }
383 } 383 }
384 384
385 return p.rules, p.GetError(scanner.NoMultiples) 385 return p.rules, p.GetError(scanner.NoMultiples)
386 } 386 }
LEFTRIGHT

Powered by Google App Engine
RSS Feeds Recent Issues | This issue
This is Rietveld f62528b