code review 7326053: misc/dashboard/builder: various cleanups

misc/dashboard/builder/main.go

 // Copyright 2011 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 package main
 
 import (
         "bytes"
         "flag"
         "fmt"
@@ skipping 22 matching lines @@
 var extraEnv = []string{
         "CC",
         "GOARM",
         "GOHOSTARCH",
         "GOHOSTOS",
         "PATH",
         "TMPDIR",
 }
 
 type Builder struct {
+        goroot       *Repo
         name         string
         goos, goarch string
         key          string
 }
 
 var (
         buildroot      = flag.String("buildroot", defaultBuildRoot(), "Directory under which to build")
         dashboard      = flag.String("dashboard", "build.golang.org", "Go Dashboard Host")
         buildRelease   = flag.Bool("release", false, "Build and upload binary release archives")
         buildRevision  = flag.String("rev", "", "Build specified revision and exit")
         buildCmd       = flag.String("cmd", filepath.Join(".", allCmd), "Build command (specify relative to go/src/)")
         failAll        = flag.Bool("fail", false, "fail all builds")
         parallel       = flag.Bool("parallel", false, "Build multiple targets in parallel")
         buildTimeout   = flag.Duration("buildTimeout", 60*time.Minute, "Maximum time to wait for builds and tests")
         cmdTimeout     = flag.Duration("cmdTimeout", 5*time.Minute, "Maximum time to wait for an external command")
         commitInterval = flag.Duration("commitInterval", 1*time.Minute, "Time to wait between polling for new commits (0 disables commit poller)")
         verbose        = flag.Bool("v", false, "verbose")
 )
 
 var (
         binaryTagRe = regexp.MustCompile(`^(release\.r|weekly\.)[0-9\-.]+`)
         releaseRe   = regexp.MustCompile(`^release\.r[0-9\-.]+`)
         allCmd      = "all" + suffix
         cleanCmd    = "clean" + suffix
         suffix      = defaultSuffix()
-
-        goroot *Repo
 )
 
 func main() {
         flag.Usage = func() {
                 fmt.Fprintf(os.Stderr, "usage: %s goos-goarch...\n", os.Args[0])
                 flag.PrintDefaults()
                 os.Exit(2)
         }
         flag.Parse()
         if len(flag.Args()) == 0 {
                 flag.Usage()
         }
-        goroot = &Repo{
+        goroot := &Repo{
                 Path: filepath.Join(*buildroot, "goroot"),
         }
 
         // set up work environment, use existing enviroment if possible
         if goroot.Exists() {
                 log.Print("Found old workspace, will use it")
         } else {
                 if err := os.RemoveAll(*buildroot); err != nil {
                         log.Fatalf("Error removing build root (%s): %s", *buildroot, err)
                 }
                 if err := os.Mkdir(*buildroot, mkdirPerm); err != nil {
                         log.Fatalf("Error making build root (%s): %s", *buildroot, err)
                 }
                 var err error
                 goroot, err = RemoteRepo(hgUrl).Clone(goroot.Path, "tip")
                 if err != nil {
                         log.Fatal("Error cloning repository:", err)
                 }
         }
 
         // set up builders
         builders := make([]*Builder, len(flag.Args()))
-        for i, builder := range flag.Args() {
-                b, err := NewBuilder(builder)
+        for i, name := range flag.Args() {
+                b, err := NewBuilder(goroot, name)
                 if err != nil {
                         log.Fatal(err)
                 }
                 builders[i] = b
         }
 
         if *failAll {
                 failMode(builders)
                 return
         }
 
         // if specified, build revision and return
         if *buildRevision != "" {
                 hash, err := goroot.FullHash(*buildRevision)
                 if err != nil {
                         log.Fatal("Error finding revision: ", err)
                 }
                 for _, b := range builders {
                         if err := b.buildHash(hash); err != nil {
                                 log.Println(err)
                         }
                 }
                 return
         }
 
         // Start commit watcher
-        go commitWatcher()
+        go commitWatcher(goroot)
 
         // go continuous build mode
         // check for new commits and build them
         for {
                 built := false
                 t := time.Now()
                 if *parallel {
                         done := make(chan bool)
                         for _, b := range builders {
                                 go func(b *Builder) {
@@ skipping 28 matching lines @@
                 for _, b := range builders {
                         built = b.failBuild() || built
                 }
                 // stop if there was nothing to fail
                 if !built {
                         break
                 }
         }
 }
 
-func NewBuilder(builder string) (*Builder, error) {
-        b := &Builder{name: builder}
+func NewBuilder(goroot *Repo, name string) (*Builder, error) {
+        b := &Builder{
+                goroot: goroot,
+                name:   name,
+        }
 
         // get goos/goarch from builder string
-        s := strings.SplitN(builder, "-", 3)
+        s := strings.SplitN(b.name, "-", 3)
         if len(s) >= 2 {
                 b.goos, b.goarch = s[0], s[1]
         } else {
-                return nil, fmt.Errorf("unsupported builder form: %s", builder)
+                return nil, fmt.Errorf("unsupported builder form: %s", name)
         }
 
         // read keys from keyfile
         fn := ""
         if runtime.GOOS == "windows" {
                 fn = os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
         } else {
                 fn = os.Getenv("HOME")
         }
         fn = filepath.Join(fn, ".gobuildkey")
@@ skipping 14 matching lines @@
 func (b *Builder) build() bool {
         hash, err := b.todo("build-go-commit", "", "")
         if err != nil {
                 log.Println(err)
                 return false
         }
         if hash == "" {
                 return false
         }
 
-        // Look for hash locally before running hg pull.
-        log.Println("looking for hash", hash)
-        if _, err := goroot.FullHash(hash[:12]); err != nil {
-                // Don't have hash, so run hg pull.
-                log.Printf("Repo: %v does not have hash %s, pulling", goroot, hash)
-                if err := goroot.Pull(); err != nil {
-                        log.Println("hg pull failed:", err)
-                        return false
-                }
-        }
-
-        err = b.buildHash(hash)
-        if err != nil {
+        if err := b.buildHash(hash); err != nil {
                 log.Println(err)
         }
         return true
 }
 
 func (b *Builder) buildHash(hash string) error {
         log.Println(b.name, "building", hash)
 
         // create place in which to do work
         workpath := filepath.Join(*buildroot, b.name+"-"+hash[:12])
         if err := os.Mkdir(workpath, mkdirPerm); err != nil {
                 return err
         }
         defer os.RemoveAll(workpath)
 
         // pull before cloning to ensure we have the revision
-        if err := goroot.Pull(); err != nil {
+        if err := b.goroot.Pull(); err != nil {
                 return err
         }
 
         // clone repo at specified revision
-        if _, err := goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
+        if _, err := b.goroot.Clone(filepath.Join(workpath, "go"), hash); err != nil {
                 return err
         }
 
         srcDir := filepath.Join(workpath, "go", "src")
 
         // build
         var buildlog bytes.Buffer
         logfile := filepath.Join(workpath, "build.log")
         f, err := os.Create(logfile)
         if err != nil {
@@ skipping 191 matching lines @@
         s, err := os.Stat(name)
         return err == nil && s.IsDir()
 }
 
 func isFile(name string) bool {
         s, err := os.Stat(name)
         return err == nil && !s.IsDir()
 }
 
 // commitWatcher polls hg for new commits and tells the dashboard about them.
-func commitWatcher() {
+func commitWatcher(goroot *Repo) {
         if *commitInterval == 0 {
                 log.Printf("commitInterval is %s, disabling commitWatcher", *commitInterval)
                 return
         }
         // Create builder just to get master key.
-        b, err := NewBuilder("mercurial-commit")
+        b, err := NewBuilder(goroot, "mercurial-commit")
         if err != nil {
                 log.Fatal(err)
         }
         key := b.key
 
         for {
                 if *verbose {
                         log.Printf("poll...")
                 }
                 // Main Go repository.
-                commitPoll(key, "")
+                commitPoll(goroot, "", key)
                 // Go sub-repositories.
                 for _, pkg := range dashboardPackages("subrepo") {
-                        commitPoll(key, pkg)
+                        pkgroot := &Repo{
+                                Path: filepath.Join(*buildroot, pkg),
+                        }
+                        commitPoll(pkgroot, pkg, key)
                 }
                 if *verbose {
                         log.Printf("sleep...")
                 }
                 time.Sleep(*commitInterval)
         }
 }
 
+// logByHash is a cache of all Mercurial revisions we know about,
+// indexed by full hash.
+var logByHash = map[string]*HgLog{}
+
 // commitPoll pulls any new revisions from the hg server
 // and tells the server about them.
-func commitPoll(key, pkg string) {
-        pkgRoot := goroot
-
-        if pkg != "" {
-                pkgRoot = &Repo{
-                        Path: filepath.Join(*buildroot, pkg),
-                }
-                if !pkgRoot.Exists() {
-                        var err error
-                        pkgRoot, err = RemoteRepo(repoURL(pkg)).Clone(pkgRoot.Path, "tip")
-                        if err != nil {
-                                log.Printf("%s: hg clone failed: %v", pkg, err)
-                                if err := os.RemoveAll(pkgRoot.Path); err != nil {
-                                        log.Printf("%s: %v", pkg, err)
-                                }
-                                return
+func commitPoll(repo *Repo, pkg, key string) {
+        if !repo.Exists() {
+                var err error
+                repo, err = RemoteRepo(repoURL(pkg)).Clone(repo.Path, "tip")
+                if err != nil {
+                        log.Printf("%s: hg clone failed: %v", pkg, err)
+                        if err := os.RemoveAll(repo.Path); err != nil {
+                                log.Printf("%s: %v", pkg, err)
                         }
                 }
-        }
-
-        if err := pkgRoot.Pull(); err != nil {
-                log.Printf("hg pull: %v", err)
                 return
         }
 
-        logs, err := pkgRoot.Log()
+        logs, err := repo.Log() // repo.Log calls repo.Pull internally
         if err != nil {
                 log.Printf("hg log: %v", err)
                 return
         }
 
         // Pass 1.  Fill in parents and add new log entries to logsByHash.
         // Empty parent means take parent from next log entry.
         // Non-empty parent has form 1234:hashhashhash; we want full hash.
         for i := range logs {
                 l := &logs[i]
                 if l.Parent == "" && i+1 < len(logs) {
                         l.Parent = logs[i+1].Hash
                 } else if l.Parent != "" {
-                        l.Parent, _ = pkgRoot.FullHash(l.Parent)
+                        l.Parent, _ = repo.FullHash(l.Parent)
                 }
                 if *verbose {
                         log.Printf("hg log %s: %s < %s\n", pkg, l.Hash, l.Parent)
                 }
                 if logByHash[l.Hash] == nil {
                         // Make copy to avoid pinning entire slice when only one entry is new.
                         t := *l
                         logByHash[t.Hash] = &t
                 }
         }
 
-        for i := range logs {
-                l := &logs[i]
+        for _, l := range logs {
                 addCommit(pkg, l.Hash, key)
         }
 }
 
 // addCommit adds the commit with the named hash to the dashboard.
 // key is the secret key for authentication to the dashboard.
 // It avoids duplicate effort.
 func addCommit(pkg, hash, key string) bool {
         l := logByHash[hash]
         if l == nil {
@@ skipping 70 matching lines @@
                 return v, true
         }
         keq := k + "="
         for _, kv := range os.Environ() {
                 if kv == keq {
                         return "", true
                 }
         }
         return "", false
 }