code review 4186055: html: tokenize HTML comments.

src/pkg/html/token_test.go

 // Copyright 2010 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package html
 
 import (
         "bytes"
         "os"
+        "strings"
         "testing"
 )
 
 type tokenTest struct {
         // A short description of the test case.
         desc string
         // The HTML to parse.
         html string
-        // The string representations of the expected tokens.
-        tokens []string
+        // The string representations of the expected tokens, joined by '$'.
+        golden string
 }
 
 var tokenTests = []tokenTest{
         // A single text node. The tokenizer should not break text nodes on whitespace,
         // nor should it normalize whitespace within a text node.
         {
                 "text",
                 "foo  bar",
-                []string{
-                        "foo  bar",
-                },
+                "foo  bar",
         },
         // An entity.
         {
                 "entity",
                 "one < two",
-                []string{
-                        "one < two",
-                },
+                "one < two",
         },
         // A start, self-closing and end tag. The tokenizer does not care if the start
         // and end tokens don't match; that is the job of the parser.
         {
                 "tags",
                 "<a>b<c/>d</e>",
-                []string{
-                        "<a>",
-                        "b",
-                        "<c/>",
-                        "d",
-                        "</e>",
-                },
-        },
-        // A comment.
-        {
-                "comment",
+                "<a>$b$<c/>$d$</e>",
+        },
+        // Comments.
+        {
+                "comment0",
                 "abc<b><!-- skipme --></b>def",
-                []string{
-                        "abc",
-                        "<b>",
-                        "</b>",
-                        "def",
-                },
+                "abc$<b>$</b>$def",
+        },
+        {
+                "comment1",
+                "a<!-->z",
+                "a$z",
+        },
+        {
+                "comment2",
+                "a<!--->z",
+                "a$z",
+        },
+        {
+                "comment3",
+                "a<!--x>-->z",
+                "a$z",
+        },
+        {
+                "comment4",
+                "a<!--x->-->z",
+                "a$z",
+        },
+        {
+                "comment5",
+                "a<!>z",
+                "a$&lt;!&gt;z",
+        },
+        {
+                "comment6",
+                "a<!->z",
+                "a$&lt;!-&gt;z",
+        },
+        {
+                "comment7",
+                "a<!---<>z",
+                "a$&lt;!---&lt;&gt;z",
+        },
+        {
+                "comment8",
+                "a<!--z",
+                "a$&lt;!--z",
         },
         // An attribute with a backslash.
         {
                 "backslash",
                 `<p id="a\"b">`,
-                []string{
-                        `<p id="a&quot;b">`,
-                },
+                `<p id="a&quot;b">`,
         },
         // Entities, tag name and attribute key lower-casing, and whitespace
         // normalization within a tag.
         {
                 "tricky",
                 "<p \t\n iD=\"a&quot;B\"  foo=\"bar\"><EM>te&lt;&amp;;xt</em></p>",
-                []string{
-                        `<p id="a&quot;B" foo="bar">`,
-                        "<em>",
-                        "te&lt;&amp;;xt",
-                        "</em>",
-                        "</p>",
-                },
+                `<p id="a&quot;B" foo="bar">$<em>$te&lt;&amp;;xt$</em>$</p>`,
         },
         // A non-existant entity. Tokenizing and converting back to a string should
         // escape the "&" to become "&amp;".
         {
                 "noSuchEntity",
                 `<a b="c&noSuchEntity;d">&lt;&alsoDoesntExist;&`,
-                []string{
-                        `<a b="c&amp;noSuchEntity;d">`,
-                        "&lt;&amp;alsoDoesntExist;&amp;",
-                },
+                `<a b="c&amp;noSuchEntity;d">$&lt;&amp;alsoDoesntExist;&amp;`,
         },
 }
 
 func TestTokenizer(t *testing.T) {
 loop:
         for _, tt := range tokenTests {
                 z := NewTokenizer(bytes.NewBuffer([]byte(tt.html)))
-                for i, s := range tt.tokens {
+                for i, s := range strings.Split(tt.golden, "$", -1) {
                         if z.Next() == ErrorToken {
                                 t.Errorf("%s token %d: want %q got error %v", tt.desc, i, s, z.Error())
                                 continue loop
                         }
                         actual := z.Token().String()
                         if s != actual {
                                 t.Errorf("%s token %d: want %q got %q", tt.desc, i, s, actual)
                                 continue loop
                         }
                 }
@@ skipping 121 matching lines @@
                                 }
                         }
                 }
         }
         u := "14567"
         v := string(result.Bytes())
         if u != v {
                 t.Errorf("TestBufAPI: want %q got %q", u, v)
         }
 }