allow updating dependencies from multiple files

godeps.go

 package main
 
 import (
         "bufio"
         "bytes"
         "encoding/hex"
         "errors"
         "flag"
         "fmt"
         "go/build"
         "os"
         "os/exec"
         "path/filepath"
         "regexp"
         "sort"
         "strconv"
         "strings"
         "sync"
         "time"
 
         "github.com/kisielk/gotool"
 )
 
 var (
         revFile       = flag.String("u", "", "update dependencies")
+        updateMany    = flag.Bool("U", false, "update many dependencies")
         testDeps      = flag.Bool("t", false, "include testing dependencies")
         printCommands = flag.Bool("x", false, "show executed commands")
         dryRun        = flag.Bool("n", false, "print but do not execute update commands")
         _             = flag.Bool("f", true, "(deprecated, superceded by -F) when updating, try to fetch deps if the update fails")
         noFetch       = flag.Bool("F", false, "when updating, do not try to fetch deps if the update fails")
         parallel      = flag.Int("P", 1, "max number of concurrent updates")
 )
 
 var exitCode = 0
 
 var buildContext = build.Default
 
 var usage = `
 Usage:
         godeps [flags] [pkg ...]
         godeps -u file [flags]
+        godeps -U [flags] file...

[mattyw, 2015/01/27 10:51:26]

There should be more information about how it works in the usage message.

e.g.

-U takes multiple dependency files, any common dependcies will be updated to the
latest revision

[rog, 2015/01/27 17:59:55]

Done.

 
 In the first form of usage (without the -u flag), godeps prints to
 standard output a list of all the source dependencies of the named
 packages (or the package in the current directory if none is given).
 If there is ambiguity in the source-control systems used, godeps will
 print all the available versions and an error, exiting with a false
 status. It is up to the user to remove lines from the output to make
 the output suitable for input to godeps -u.
 
 In the second form, godeps updates source to versions specified by
 the -u file argument, which should hold version information in the
 same form printed by godeps. It is an error if the file contains more
 than one line for the same package root. If a specified revision is not
 currently available, godeps will attempt to fetch it, unless the -F flag
 is provided.
 `[1:]
 
 func main() {
         flag.Usage = func() {
                 fmt.Fprintf(os.Stderr, "%s\n", usage)
                 flag.PrintDefaults()
                 os.Exit(2)
         }
         flag.Parse()
-        if *revFile != "" {
-                if flag.NArg() != 0 {
-                        flag.Usage()
+        if *revFile != "" || *updateMany {
+                var files []string
+                if *revFile != "" {
+                        if flag.NArg() != 0 {
+                                flag.Usage()
+                        }
+                        files = []string{*revFile}
+                } else {
+                        if flag.NArg() == 0 {
+                                flag.Usage()
+                        }
+                        files = flag.Args()
                 }
-                update(*revFile)
+                update(files)
         } else {
                 pkgs := flag.Args()
                 if len(pkgs) == 0 {
                         pkgs = []string{"."}
                 }
                 pkgs = gotool.ImportPaths(pkgs)
                 for _, info := range list(pkgs, *testDeps) {
                         fmt.Println(info)
                 }
         }
         os.Exit(exitCode)
 }
 
-func update(file string) {
-        projects, err := parseDepFile(file)
-        if err != nil {
-                errorf("cannot parse %q: %v", file, err)
-                return
+func update(files []string) {
+        fp := make(map[string]map[string]*depInfo)
+        for _, file := range files {
+                projects, err := parseDepFile(file)
+                if err != nil {
+                        errorf("cannot parse %q: %v", file, err)
+                        return
+                }
+                fp[file] = projects
         }
+        projects := reconcileDeps(fp)
         // First get info on all the projects, make sure their working
         // directories are all clean and prune out the ones which
         // don't need updating.
         for proj, info := range projects {
                 if info.notThere {
                         if *noFetch {
                                 errorf("%q does not exist", proj)
                                 delete(projects, proj)
                         }
                         continue
@@ skipping 10 matching lines @@
                         continue
                 }
                 if currentInfo.revid == info.revid {
                         // No need to update.
                         delete(projects, proj)
                 }
         }
         updateProjects(projects)
 }
 
+type fileDepInfo struct {
+        file string
+        *depInfo
+}
+
+// reconcileDeps takes dependencies from a bunch
+// of different dependency files and returns a single
+// unified project-to-info map containing all the dependencies,
+// each one at the latest version found.
+func reconcileDeps(fp map[string]map[string]*depInfo) map[string]*depInfo {
+        m := make(map[string]byLatestDep)
+        for file, projects := range fp {
+                for _, info := range projects {
+                        m[info.project] = append(m[info.project], fileDepInfo{
+                                file:    file,
+                                depInfo: info,
+                        })
+                }
+        }
+        r := make(map[string]*depInfo)
+        for project, infos := range m {
+                sort.Sort(infos)
+                r[project] = infos[0].depInfo
+                for _, info := range infos {
+                        if !info.notThere && info.revid != infos[0].revid {
+                                fmt.Printf("warning: ignoring older revno %q for %s in %s\n", info.revno, project, info.file)
+                        }
+                }
+        }
+        return r
+}
+
+// byLatestDep can be used to sort dependency infos
+// into latest-first order.
+type byLatestDep []fileDepInfo
+
+func (s byLatestDep) Len() int      { return len(s) }
+func (s byLatestDep) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+func (s byLatestDep) Less(i, j int) bool {
+        i0, i1 := s[i], s[j]
+        if i0.revid != i1.revid {
+                if i0.vcs.Kind() != i1.vcs.Kind() {
+                        return i0.vcs.Kind() < i1.vcs.Kind()
+                }
+                return i0.vcs.Later(i0.depInfo.VCSInfo, i1.depInfo.VCSInfo)
+        }
+        return i0.file < i1.file
+}
+
 func updateProjects(projects map[string]*depInfo) {
         limit := make(chan struct{}, *parallel)
         type result struct {
                 info *depInfo
                 err  error
         }
         results := make(chan result)
         for _, info := range projects {
                 info := info
                 go func() {
@@ skipping 379 matching lines @@
                 // testing dependencies are not transitively included.
                 ctxt.walkDeps(impPath, false)
         }
 }
 
 type VCS interface {
         Kind() string
         Info(dir string) (VCSInfo, error)
         Update(dir, revid string) error
         Fetch(dir string) error
+        Later(i0, i1 VCSInfo) bool
 }
 
 type VCSInfo struct {
         revid string
         revno string // optional
         clean bool
 }
 
 var metadataDirs = map[string]VCS{
         ".bzr": bzrVCS{},
@@ skipping 51 matching lines @@
 func (gitVCS) Update(dir string, revid string) error {
         _, err := runUpdateCmd(dir, "git", "checkout", revid)
         return err
 }
 
 func (gitVCS) Fetch(dir string) error {
         _, err := runCmd(dir, "git", "fetch")
         return err
 }
 
+func (gitVCS) Later(i0, i1 VCSInfo) bool {
+        if i0.revno == "" || i1.revno == "" {
+                return i0.revno != ""
+        }
+        t0, err0 := time.Parse(time.RFC3339, i0.revno)
+        if err0 != nil {
+                fmt.Printf("warning: cannot parse time stamp %s: %v", i0.revno, err0)
+        }
+        t1, err1 := time.Parse(time.RFC3339, i1.revno)
+        if err1 != nil {
+                fmt.Printf("warning: cannot parse time stamp %s: %v", i1.revno, err1)
+        }
+        if err0 != nil || err1 != nil {
+                return err0 == nil
+        }
+        return t0.After(t1)
+}
+
 type bzrVCS struct{}
 
 func (bzrVCS) Kind() string {
         return "bzr"
 }
 
 var validBzrInfo = regexp.MustCompile(`^([0-9]+) ([^ \t]+)$`)
 var shelveLine = regexp.MustCompile(`^[0-9]+ (shelves exist|shelf exists)\.`)
 
 func (bzrVCS) Info(dir string) (VCSInfo, error) {
@@ skipping 29 matching lines @@
 func (bzrVCS) Update(dir string, revid string) error {
         _, err := runUpdateCmd(dir, "bzr", "update", "-r", "revid:"+revid)
         return err
 }
 
 func (bzrVCS) Fetch(dir string) error {
         _, err := runCmd(dir, "bzr", "pull")
         return err
 }
 
+func (bzrVCS) Later(i0, i1 VCSInfo) bool {
+        return laterIntRevno(i0, i1)
+}
+
 var validHgInfo = regexp.MustCompile(`^([a-f0-9]+) ([0-9]+)$`)
 
 type hgVCS struct{}
 
 func (hgVCS) Info(dir string) (VCSInfo, error) {
         out, err := runCmd(dir, "hg", "log", "-l", "1", "-r", ".", "--template", "{node} {rev}")
         if err != nil {
                 return VCSInfo{}, err
         }
         m := validHgInfo.FindStringSubmatch(strings.TrimSpace(out))
@@ skipping 19 matching lines @@
 func (hgVCS) Update(dir string, revid string) error {
         _, err := runUpdateCmd(dir, "hg", "update", revid)
         return err
 }
 
 func (hgVCS) Fetch(dir string) error {
         _, err := runCmd(dir, "hg", "pull")
         return err
 }
 
+func (hgVCS) Later(i0, i1 VCSInfo) bool {
+        return laterIntRevno(i0, i1)
+}
+
+func laterIntRevno(i0, i1 VCSInfo) bool {
+        if i0.revno == "" || i1.revno == "" {
+                return i0.revno != ""
+        }
+        r0, err0 := strconv.Atoi(i0.revno)
+        if err0 != nil {
+                fmt.Printf("warning: cannot parse revision number %s: %v", i0.revno, err0)

[mattyw, 2015/01/27 10:51:26]

So if we can't parse a revision id we always assume that i0 is the latest. What
situations is this likely to fail? I can't help but think we should return an
error at this point. We then have the choice of stopping the whole thing - or
skipping this project and printing a warning. It feels a bit consistent
otherwise.
If we left it like this the usage would be:

-U takes multiple dependency files, any common dependcies will be updated to the
latest revision. If a particular revid can't be parsed we assume the first revid
we find is the one we want. This means the list of dependency files can be
effectively ordered from most important -> least important

[rog, 2015/01/27 17:59:55]

That's not quite right. If we can't parse a revision id, we assume that
the one that parses OK is the latest. If we can't parse either of them,
we always return false.
I think this is a good idea. I have now changed the behaviour so that by default
it uses the earliest revision file stated, which makes it easy to update a bunch
of projects while keeping deps for a particular project intact.

Unfortunately if this is desired (for example we want to give juju-core deps
precedence), there's nothing to stop the highest-precedence project itself being
updated. We have
potential cyclic project dependencies here, and updating dependencies can
affect the dependency graph itself. Some thought is required.

I've also added a -latest flag to enable the previous behaviour (updating
to latest dependencies), which will also be useful, I think.

+        }
+        r1, err1 := strconv.Atoi(i1.revno)
+        if err1 != nil {
+                fmt.Printf("warning: cannot parse revision number %s: %v", i1.revno, err1)
+        }
+        if err0 != nil || err1 != nil {
+                return err0 == nil
+        }
+        return r0 > r1
+}
+
 func runUpdateCmd(dir string, name string, args ...string) (string, error) {
         if *dryRun {
                 printShellCommand(dir, name, args)
                 return "", nil
         }
         return runCmd(dir, name, args...)
 }
 
 func runCmd(dir string, name string, args ...string) (string, error) {
         var outData, errData bytes.Buffer
@@ skipping 37 matching lines @@
                 buf.WriteString(" ")
                 buf.WriteString(shquote(arg))
         }
         fmt.Fprintf(os.Stderr, "%s\n", buf.Bytes())
 }
 
 func shquote(s string) string {
         // single-quote becomes single-quote, double-quote, single-quote, double-quote, single-quote
         return `'` + strings.Replace(s, `'`, `'"'"'`, -1) + `'`
 }