code review 155730043: go.tools/dashboard/watcher: commit watcher rewrite

dashboard/watcher/watcher.go

+// Command watcher watches the specified repository for new commits
+// and reports them to the build dashboard.
+package main
+
+import (
+        "bytes"
+        "encoding/json"
+        "encoding/xml"
+        "flag"
+        "fmt"
+        "io/ioutil"
+        "log"
+        "net/http"
+        "os"
+        "os/exec"
+        "path/filepath"
+        "runtime"
+        "strings"
+        "time"
+)
+
+var (
+        pkgPath      = flag.String("pkg", "", "Package path (empty for main repo)")
+        repo         = flag.String("repo", "https://code.google.com/p/go", "Repository URL")
+        local        = flag.String("local", "", "Local repo (for testing only)")
+        dashboard    = flag.String("dash", "https://build.golang.org/", "Dashboard URL")
+        keyFile      = flag.String("key", defaultKeyFile, "Build dashboard key file")
+        pollInterval = flag.Duration("poll", 10*time.Second, "Remote repo poll interval")
+)
+
+var (
+        defaultKeyFile = filepath.Join(homeDir(), ".gobuildkey")
+        dashboardStart = "" // we'll ignore commits before this hash
+        dashboardKey   = ""
+)
+
+func main() {
+        flag.Parse()
+        if *pkgPath == "" {
+                // We only started building the main repo at this hash.
+                dashboardStart = "2f970046e1ba96f32de62f5639b7141cda2e977c"
+        }
+        if err := run(); err != nil {
+                fmt.Fprintln(os.Stderr, err)
+                os.Exit(1)
+        }
+}
+
+func run() error {
+        if k, err := readKey(); err != nil {
+                return err
+        } else {
+                dashboardKey = k
+        }
+
+        var goroot string
+        if *local != "" {
+                goroot = *local
+        } else {
+                dir, err := ioutil.TempDir("", "watcher")
+                if err != nil {
+                        return err
+                }
+                defer os.RemoveAll(dir)
+                goroot = filepath.Join(dir, "go")
+                cmd := exec.Command("hg", "clone", *repo, goroot)
+                if out, err := cmd.CombinedOutput(); err != nil {
+                        return fmt.Errorf("%v\n\n%s", err, out)
+                }
+        }
+
+        commits, err := allCommits(goroot)
+        if err != nil {
+                return err
+        }
+
+        branches := make(map[string]*Branch)
+        for _, c := range commits {
+                if c.children == nil {
+                        if !validHead(c) {
+                                continue
+                        }
+                        seen, err := lastSeen(commits, c.Hash)
+                        if err != nil {
+                                return err
+                        }
+                        b := &Branch{Name: c.Branch, Head: c, LastSeen: seen}
+                        branches[c.Branch] = b
+                        log.Printf("found branch: %v", b)
+                }
+        }
+        log.Printf("found %v branches among %v commits\n", len(branches), len(commits))
+
+        for {
+                if err := update(goroot, commits, branches); err != nil {
+                        return err
+                }
+                for _, b := range branches {
+                        if b.Head == b.LastSeen {
+                                continue
+                        }
+                        c := b.LastSeen
+                        if c == nil {
+                                // Haven't seen any: find the earliest commit on this branch.
+                                for c := b.Head; c.Branch == b.Name; c = c.parent {
+                                }
+                        }
+                        // Add unseen commits on this branch, working forward from last seen.
+                        for c.children != nil {
+                                var next *Commit
+                                for _, c2 := range c.children {
+                                        if c2.Branch != b.Name {
+                                                continue
+                                        }
+                                        if next != nil {
+                                                // Shouldn't happen, but be paranoid.
+                                                return fmt.Errorf("found multiple children on branch %q: %v and %v", b.Name, next, c2)
+                                        }
+                                        next = c2
+                                }
+                                if next == nil {
+                                        // Hit the head commit, nothing more to do.
+                                        continue
+                                }
+                                c = next
+
+                                if err := postCommit(c); err != nil {
+                                        return err
+                                }
+                                b.LastSeen = c
+                        }
+                }
+                time.Sleep(*pollInterval)
+        }
+}
+
+// validHead reports whether the specified commit should be considered a branch
+// head. It considers pre-go1 branches and certain specific commits as invalid.
+func validHead(c *Commit) bool {
+        if strings.HasPrefix(c.Branch, "release-branch.r") {
+                return true
+        }
+        // Not sure why these revisions have no child commits,
+        // but they're old so let's just ignore them.
+        return c.Hash == "b59f4ff1b51094314f735a4d57a2b8f06cfadf15" ||
+                c.Hash == "fc75f13840b896e82b9fa6165cf705fbacaf019c"
+}
+
+// lastSeen finds the most recent commit the dashboard has seen,
+// starting at the specified head.
+func lastSeen(commits map[string]*Commit, head string) (*Commit, error) {
+        h, ok := commits[head]
+        if !ok {
+                return nil, fmt.Errorf("lastSeen: can't find %q in commits", head)
+        }
+
+        var s []*Commit
+        for c := h; c != nil; c = c.parent {
+                s = append(s, c)
+                if c.Hash == dashboardStart {
+                        break
+                }
+        }
+
+        for _, c := range s {
+                u := *dashboard + "commit?hash=" + c.Hash
+                r, err := http.Get(u)
+                if err != nil {
+                        return nil, err
+                }
+                var resp struct {
+                        Error string
+                }
+                err = json.NewDecoder(r.Body).Decode(&resp)
+                r.Body.Close()
+                if err != nil {
+                        return nil, err
+                }
+                if resp.Error == "" {
+                        return c, nil
+                }
+        }
+        return nil, nil
+}
+
+// postCommit sends a commit to the build dashboard.
+func postCommit(c *Commit) error {
+        log.Printf("sending commit to dashboard: %v", c)
+
+        t, err := time.Parse(time.RFC3339, c.Date)
+        if err != nil {
+                return err
+        }
+        dc := struct {
+                PackagePath string // (empty for main repo commits)
+                Hash        string
+                ParentHash  string
+
+                User string
+                Desc string
+                Time time.Time
+
+                NeedsBenchmarking bool
+        }{
+                *pkgPath,
+                c.Hash,
+                c.Parent,
+                c.Author,
+                c.Desc,
+                t,
+                true,
+        }
+        b, err := json.Marshal(dc)
+        if err != nil {
+                return err
+        }
+
+        u := *dashboard + "commit?key=" + dashboardKey
+        r, err := http.Post(u, "text/json", bytes.NewReader(b))
+        if err != nil {
+                return err
+        }
+        if r.StatusCode != 200 {
+                return fmt.Errorf("status: %v", r.Status)
+        }
+        return nil
+}
+
+// allCommits runs "hg log" in goroot and returns a map of all its commits.
+func allCommits(goroot string) (map[string]*Commit, error) {
+        log, err := hgLog(goroot)
+        if err != nil {
+                return nil, err
+        }
+        commits := make(map[string]*Commit)
+        for _, c := range log {
+                commits[c.Hash] = c
+        }
+        for _, c := range commits {
+                if p, ok := commits[c.Parent]; ok {
+                        c.parent = p
+                        p.children = append(p.children, c)
+                }
+        }
+        return commits, nil
+}
+
+// update runs "hg pull" in the specified goroot,
+// looks for new commits and branches,
+// and updates the comits and branches maps.
+func update(goroot string, commits map[string]*Commit, branches map[string]*Branch) error {
+        if err := hgPull(goroot); err != nil {
+                return err
+        }
+
+        newBranches := make(map[string]*Branch)
+
+        // Check each branch for new commits.
+        for _, b := range branches {
+                // Find all commits from known head.
+                args := []string{"-r", b.Head.Hash + ":"}
+                if b.Name != "" {
+                        // Specify -b flag only for non-default branches.
+                        args = append(args, "-b", b.Name)
+                }
+                l, err := hgLog(goroot, args...)
+                if err != nil {
+                        return err
+                }
+
+                // Add unknown commits to commits, and create new branches.
+                for _, c := range l {
+                        if _, ok := commits[c.Hash]; ok {
+                                continue
+                        }
+
+                        log.Printf("found new commit %v", c)
+                        commits[c.Hash] = c
+
+                        p, ok := commits[c.Parent]
+                        if !ok {
+                                return fmt.Errorf("can't find parent hash %q for %v", c.Parent, c)
+                        }
+
+                        if c.Branch != b.Name {
+                                // The commit isn't on this branch.
+
+                                // If it's a new branch we've already seen this loop, update the new branch head.
+                                if b2, ok := newBranches[c.Branch]; ok {
+                                        log.Printf("branch %q head updated by: %v", c.Branch, c)
+                                        b2.Head = c
+                                        continue
+                                }
+
+                                // Sanity checks:
+                                if _, ok := branches[c.Branch]; ok && p.Branch != c.Branch {
+                                        // Shouldn't happen, but be paranoid about it.
+                                        return fmt.Errorf("commit branches into existing branch %q: %v", c.Branch, c)
+                                }
+                                if p.Branch == c.Branch {
+                                        // Shouldn't happen, but be paranoid.
+                                        return fmt.Errorf("commit on branch we haven't seen: %v", c)
+                                }
+
+                                // A new branch we haven't seen yet; add it.
+                                log.Printf("new branch %q created by %v", c.Branch, c)
+                                newBranches[c.Branch] = &Branch{Name: c.Branch, Head: c}
+                                continue
+                        }
+
+                        // Just a commit to a known branch, add it and update the branch head.
+                        c.parent = p
+                        p.children = append(p.children, c)
+                        b.Head = c
+                }
+        }
+
+        // Find latest dashboard commits for new branches.
+        for _, b := range newBranches {
+                seen, err := lastSeen(commits, b.Head.Hash)
+                if err != nil {
+                        return err
+                }
+                b.LastSeen = seen
+                branches[b.Name] = b
+        }
+
+        return nil
+}
+
+// hgLog runs "hg log" with the supplied arguments
+// and parses the output into Commit values.
+func hgLog(dir string, args ...string) ([]*Commit, error) {
+        args = append([]string{"log", "--template", xmlLogTemplate}, args...)
+        cmd := exec.Command("hg", args...)
+        cmd.Dir = dir
+        out, err := cmd.CombinedOutput()
+        if err != nil {
+                return nil, err
+        }
+
+        // We have a commit with description that contains 0x1b byte.
+        // Mercurial does not escape it, but xml.Unmarshal does not accept it.
+        out = bytes.Replace(out, []byte{0x1b}, []byte{'?'}, -1)
+
+        var logStruct struct {
+                Log []*Commit
+        }
+        err = xml.Unmarshal([]byte("<Top>"+string(out)+"</Top>"), &logStruct)
+        if err != nil {
+                return nil, err
+        }
+        return logStruct.Log, nil
+}
+
+// hgPull runs "hg pull" in the specified directory.
+func hgPull(dir string) error {
+        cmd := exec.Command("hg", "pull")
+        cmd.Dir = dir
+        if out, err := cmd.CombinedOutput(); err != nil {
+                return fmt.Errorf("%v\n\n%s", err, out)
+        }
+        return nil
+}
+
+// Branch represents a Mercurial branch.
+type Branch struct {
+        Name     string
+        Head     *Commit
+        LastSeen *Commit // the last commit posted to the dashboard
+}
+
+func (b *Branch) String() string {
+        return fmt.Sprintf("%q(Head: %v LastSeen: %v)", b.Name, b.Head, b.LastSeen)
+}
+
+// Commit represents a single Mercurial revision.
+type Commit struct {
+        Hash   string
+        Author string
+        Date   string
+        Desc   string
+        Parent string
+        Branch string
+        Files  string
+
+        // For walking the graph.
+        parent   *Commit
+        children []*Commit
+}
+
+func (c *Commit) String() string {
+        return fmt.Sprintf("%v(%q)", c.Hash, strings.SplitN(c.Desc, "\n", 2)[0])
+}
+
+// xmlLogTemplate is a template to pass to Mercurial to make
+// hg log print the log in valid XML for parsing with xml.Unmarshal.
+// Can not escape branches and files, because it crashes python with:
+// AttributeError: 'NoneType' object has no attribute 'replace'
+const xmlLogTemplate = `
+        <Log>
+        <Hash>{node|escape}</Hash>
+        <Parent>{p1node}</Parent>
+        <Author>{author|escape}</Author>
+        <Date>{date|rfc3339date}</Date>
+        <Desc>{desc|escape}</Desc>
+        <Branch>{branches}</Branch>
+        <Files>{files}</Files>
+        </Log>
+`
+
+func homeDir() string {
+        switch runtime.GOOS {
+        case "plan9":
+                return os.Getenv("home")
+        case "windows":
+                return os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
+        }
+        return os.Getenv("HOME")
+}
+
+func readKey() (string, error) {
+        c, err := ioutil.ReadFile(*keyFile)
+        if err != nil {
+                return "", err
+        }
+        return string(bytes.TrimSpace(bytes.SplitN(c, []byte("\n"), 2)[0])), nil
+}