Fix tools metadata generation for null provider

environs/tools/simplestreams_test.go

 // Copyright 2013 Canonical Ltd.
 // Licensed under the AGPLv3, see LICENCE file for details.
 
 package tools_test
 
 import (
         "flag"
         "fmt"
-        "path/filepath"
+        "io"
         "reflect"
         "testing"
 
         "launchpad.net/goamz/aws"
         gc "launchpad.net/gocheck"
 
         "launchpad.net/juju-core/environs/filestorage"
         "launchpad.net/juju-core/environs/simplestreams"
         sstesting "launchpad.net/juju-core/environs/simplestreams/testing"
+        "launchpad.net/juju-core/environs/storage"
         "launchpad.net/juju-core/environs/tools"
         ttesting "launchpad.net/juju-core/environs/tools/testing"
+        "launchpad.net/juju-core/testing/testbase"
         coretools "launchpad.net/juju-core/tools"
         "launchpad.net/juju-core/version"
 )
 
 var live = flag.Bool("live", false, "Include live simplestreams tests")
 var vendor = flag.String("vendor", "", "The vendor representing the source of the simplestream data")
 
 type liveTestData struct {
         baseURL        string
         requireSigned  bool
@@ skipping 237 matching lines @@
                 Size:     2973595,
                 Path:     "mirrored-path/juju-1.13.0-precise-amd64.tgz",
                 FullPath: "test:/mirrored-path/juju-1.13.0-precise-amd64.tgz",
                 FileType: "tar.gz",
                 SHA256:   "447aeb6a934a5eaec4f703eda4ef2dde",
         }
         c.Assert(err, gc.IsNil)
         c.Assert(toolsMetadata[0], gc.DeepEquals, expectedMetadata)
 }
 
-func assertMetadataMatches(c *gc.C, toolList coretools.List, metadata []*tools.ToolsMetadata) {
+func assertMetadataMatches(c *gc.C, storageDir string, toolList coretools.List, metadata []*tools.ToolsMetadata) {
         var expectedMetadata []*tools.ToolsMetadata = make([]*tools.ToolsMetadata, len(toolList))
         for i, tool := range toolList {
-                if tool.URL != "" {
-                        size, sha256 := ttesting.SHA256sum(c, tool.URL)
-                        tool.SHA256 = sha256
-                        tool.Size = size
-                }
                 expectedMetadata[i] = &tools.ToolsMetadata{
                         Release:  tool.Version.Series,
                         Version:  tool.Version.Number.String(),
                         Arch:     tool.Version.Arch,
                         Size:     tool.Size,
                         Path:     fmt.Sprintf("releases/juju-%s.tgz", tool.Version.String()),
                         FileType: "tar.gz",
                         SHA256:   tool.SHA256,
                 }
         }
         c.Assert(metadata, gc.DeepEquals, expectedMetadata)
 }
 
 func (s *simplestreamsSuite) TestWriteMetadataNoFetch(c *gc.C) {
         toolsList := coretools.List{
                 {
                         Version: version.MustParseBinary("1.2.3-precise-amd64"),
                         Size:    123,
                         SHA256:  "abcd",
                 }, {
                         Version: version.MustParseBinary("2.0.1-raring-amd64"),
                         Size:    456,
                         SHA256:  "xyz",
                 },
         }
         dir := c.MkDir()
         writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
         c.Assert(err, gc.IsNil)
-        err = tools.WriteMetadata(toolsList, false, writer)
+        err = tools.MergeAndWriteMetadata(writer, toolsList)
         c.Assert(err, gc.IsNil)
         metadata := ttesting.ParseMetadata(c, dir)
-        assertMetadataMatches(c, toolsList, metadata)
+        assertMetadataMatches(c, dir, toolsList, metadata)
 }
 
 func (s *simplestreamsSuite) TestWriteMetadata(c *gc.C) {
         var versionStrings = []string{
                 "1.2.3-precise-amd64",
                 "2.0.1-raring-amd64",
         }
         dir := c.MkDir()
         ttesting.MakeTools(c, dir, "releases", versionStrings)
 
         toolsList := coretools.List{
                 {
                         // If sha256/size is already known, do not recalculate
                         Version: version.MustParseBinary("1.2.3-precise-amd64"),
                         Size:    123,
                         SHA256:  "abcd",
                 }, {
                         Version: version.MustParseBinary("2.0.1-raring-amd64"),
-                        URL:     "file://" + filepath.Join(dir, "tools/releases/juju-2.0.1-raring-amd64.tgz"),
+                        // The URL is not used for generating metadata.
+                        URL: "bogus://",
                 },
         }
         writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
         c.Assert(err, gc.IsNil)
-        err = tools.WriteMetadata(toolsList, true, writer)
+        err = tools.MergeAndWriteMetadata(writer, toolsList)
         c.Assert(err, gc.IsNil)
         metadata := ttesting.ParseMetadata(c, dir)
-        assertMetadataMatches(c, toolsList, metadata)
+        assertMetadataMatches(c, dir, toolsList, metadata)
 }
 
 func (s *simplestreamsSuite) TestWriteMetadataMergeWithExisting(c *gc.C) {
         dir := c.MkDir()
         existingToolsList := coretools.List{
                 {
                         Version: version.MustParseBinary("1.2.3-precise-amd64"),
                         Size:    123,
                         SHA256:  "abc",
                 }, {
                         Version: version.MustParseBinary("2.0.1-raring-amd64"),
                         Size:    456,
                         SHA256:  "xyz",
                 },
         }
         writer, err := filestorage.NewFileStorageWriter(dir, filestorage.UseDefaultTmpDir)
         c.Assert(err, gc.IsNil)
-        err = tools.WriteMetadata(existingToolsList, true, writer)
+        err = tools.MergeAndWriteMetadata(writer, existingToolsList)
         c.Assert(err, gc.IsNil)
         newToolsList := coretools.List{
                 existingToolsList[0],
                 {
                         Version: version.MustParseBinary("2.1.0-raring-amd64"),
                         Size:    789,
                         SHA256:  "def",
                 },
         }
-        err = tools.WriteMetadata(newToolsList, true, writer)
+        err = tools.MergeAndWriteMetadata(writer, newToolsList)
         c.Assert(err, gc.IsNil)
         requiredToolsList := append(existingToolsList, newToolsList[1])
         metadata := ttesting.ParseMetadata(c, dir)
-        assertMetadataMatches(c, requiredToolsList, metadata)
+        assertMetadataMatches(c, dir, requiredToolsList, metadata)
 }
 
 type productSpecSuite struct{}
 
 var _ = gc.Suite(&productSpecSuite{})
 
 func (s *productSpecSuite) TestId(c *gc.C) {
         toolsConstraint := tools.NewVersionedToolsConstraint("1.13.0", simplestreams.LookupParams{
                 Series: []string{"precise"},
                 Arches: []string{"amd64"},
@@ skipping 80 matching lines @@
         c.Assert(product, gc.NotNil)
         c.Assert(product.Items, gc.HasLen, 1)
         version := product.Items["20133008"]
         c.Assert(version, gc.NotNil)
         c.Assert(version.Items, gc.HasLen, 1)
         item := version.Items["1.10.0-precise-amd64"]
         c.Assert(item, gc.NotNil)
         c.Assert(item, gc.FitsTypeOf, &tools.ToolsMetadata{})
         c.Assert(item.(*tools.ToolsMetadata).Size, gc.Equals, int64(9223372036854775807))
 }
+
+type metadataHelperSuite struct {
+        testbase.LoggingSuite
+}
+
+var _ = gc.Suite(&metadataHelperSuite{})
+
+func (*metadataHelperSuite) TestMetadataFromTools(c *gc.C) {
+        metadata := tools.MetadataFromTools(nil)
+        c.Assert(metadata, gc.HasLen, 0)
+
+        toolsList := coretools.List{{
+                Version: version.MustParseBinary("1.2.3-precise-amd64"),
+                Size:    123,
+                SHA256:  "abc",
+        }, {
+                Version: version.MustParseBinary("2.0.1-raring-amd64"),
+                URL:     "file:///tmp/releases/juju-2.0.1-raring-amd64.tgz",
+                Size:    456,
+                SHA256:  "xyz",
+        }}
+        metadata = tools.MetadataFromTools(toolsList)
+        c.Assert(metadata, gc.HasLen, len(toolsList))
+        for i, t := range toolsList {
+                md := metadata[i]
+                c.Assert(md.Release, gc.Equals, t.Version.Series)
+                c.Assert(md.Version, gc.Equals, t.Version.Number.String())
+                c.Assert(md.Arch, gc.Equals, t.Version.Arch)
+                c.Assert(md.FullPath, gc.Equals, t.URL)
+                c.Assert(md.Path, gc.Equals, tools.StorageName(t.Version)[len("tools/"):])
+                c.Assert(md.FileType, gc.Equals, "tar.gz")
+                c.Assert(md.Size, gc.Equals, t.Size)
+                c.Assert(md.SHA256, gc.Equals, t.SHA256)
+        }
+}
+
+type countingStorage struct {
+        storage.StorageReader
+        counter int
+}
+
+func (c *countingStorage) Get(name string) (io.ReadCloser, error) {
+        c.counter++
+        return c.StorageReader.Get(name)
+}
+
+func (*metadataHelperSuite) TestResolveMetadata(c *gc.C) {
+        var versionStrings = []string{"1.2.3-precise-amd64"}
+        dir := c.MkDir()
+        ttesting.MakeTools(c, dir, "releases", versionStrings)
+        toolsList := coretools.List{{
+                Version: version.MustParseBinary(versionStrings[0]),
+                Size:    123,
+                SHA256:  "abc",
+        }}
+
+        stor, err := filestorage.NewFileStorageReader(dir)
+        c.Assert(err, gc.IsNil)
+        err = tools.ResolveMetadata(stor, nil)
+        c.Assert(err, gc.IsNil)
+
+        // We already have size/sha256, so ensure that storage isn't consulted.
+        countingStorage := &countingStorage{StorageReader: stor}
+        metadata := tools.MetadataFromTools(toolsList)
+        err = tools.ResolveMetadata(countingStorage, metadata)
+        c.Assert(err, gc.IsNil)
+        c.Assert(countingStorage.counter, gc.Equals, 0)
+
+        // Now clear size/sha256, and check that it is called, and
+        // the size/sha256 sum are updated.
+        metadata[0].Size = 0
+        metadata[0].SHA256 = ""
+        err = tools.ResolveMetadata(countingStorage, metadata)
+        c.Assert(err, gc.IsNil)
+        c.Assert(countingStorage.counter, gc.Equals, 1)
+        c.Assert(metadata[0].Size, gc.Not(gc.Equals), 0)
+        c.Assert(metadata[0].SHA256, gc.Not(gc.Equals), "")
+}
+
+func (*metadataHelperSuite) TestMergeMetadata(c *gc.C) {
+        md1 := &tools.ToolsMetadata{
+                Release: "precise",
+                Version: "1.2.3",
+                Arch:    "amd64",
+                Path:    "path1",
+        }
+        md2 := &tools.ToolsMetadata{
+                Release: "precise",
+                Version: "1.2.3",
+                Arch:    "amd64",
+                Path:    "path2",
+        }
+        md3 := &tools.ToolsMetadata{
+                Release: "raring",
+                Version: "1.2.3",
+                Arch:    "amd64",
+                Path:    "path3",
+        }
+
+        withSize := func(md *tools.ToolsMetadata, size int64) *tools.ToolsMetadata {
+                clone := *md
+                clone.Size = size
+                return &clone
+        }
+        withSHA256 := func(md *tools.ToolsMetadata, sha256 string) *tools.ToolsMetadata {
+                clone := *md
+                clone.SHA256 = sha256
+                return &clone
+        }
+
+        type mdlist []*tools.ToolsMetadata
+        type test struct {
+                name             string
+                lhs, rhs, merged []*tools.ToolsMetadata
+                err              string
+        }
+        tests := []test{{
+                name:   "non-empty lhs, empty rhs",
+                lhs:    mdlist{md1},
+                rhs:    nil,
+                merged: mdlist{md1},
+        }, {
+                name:   "empty lhs, non-empty rhs",
+                lhs:    nil,
+                rhs:    mdlist{md2},
+                merged: mdlist{md2},
+        }, {
+                name:   "identical lhs, rhs",
+                lhs:    mdlist{md1},
+                rhs:    mdlist{md1},
+                merged: mdlist{md1},
+        }, {
+                name:   "same tools in lhs and rhs, neither have size: prefer lhs",
+                lhs:    mdlist{md1},
+                rhs:    mdlist{md2},
+                merged: mdlist{md1},
+        }, {
+                name:   "same tools in lhs and rhs, only lhs has a size: prefer lhs",
+                lhs:    mdlist{withSize(md1, 123)},
+                rhs:    mdlist{md2},
+                merged: mdlist{withSize(md1, 123)},
+        }, {
+                name:   "same tools in lhs and rhs, only rhs has a size: prefer rhs",
+                lhs:    mdlist{md1},
+                rhs:    mdlist{withSize(md2, 123)},
+                merged: mdlist{withSize(md2, 123)},
+        }, {
+                name:   "same tools in lhs and rhs, both have the same size: prefer lhs",
+                lhs:    mdlist{withSize(md1, 123)},
+                rhs:    mdlist{withSize(md2, 123)},
+                merged: mdlist{withSize(md1, 123)},
+        }, {
+                name: "same tools in lhs and rhs, both have different sizes: error",
+                lhs:  mdlist{withSize(md1, 123)},
+                rhs:  mdlist{withSize(md2, 456)},
+                err:  "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,456\\) sha256=\\(,\\)",
+        }, {
+                name: "same tools in lhs and rhs, both have same size but different sha256: error",
+                lhs:  mdlist{withSHA256(withSize(md1, 123), "a")},
+                rhs:  mdlist{withSHA256(withSize(md2, 123), "b")},
+                err:  "metadata mismatch for 1\\.2\\.3-precise-amd64: sizes=\\(123,123\\) sha256=\\(a,b\\)",
+        }, {
+                name:   "lhs is a proper superset of rhs: union of lhs and rhs",
+                lhs:    mdlist{md1, md3},
+                rhs:    mdlist{md1},
+                merged: mdlist{md1, md3},
+        }, {
+                name:   "rhs is a proper superset of lhs: union of lhs and rhs",
+                lhs:    mdlist{md1},
+                rhs:    mdlist{md1, md3},
+                merged: mdlist{md1, md3},
+        }}
+        for i, test := range tests {
+                c.Logf("test %d: %s", i, test.name)
+                merged, err := tools.MergeMetadata(test.lhs, test.rhs)
+                if test.err == "" {
+                        c.Assert(err, gc.IsNil)
+                        c.Assert(merged, gc.DeepEquals, test.merged)
+                } else {
+                        c.Assert(err, gc.ErrorMatches, test.err)
+                        c.Assert(merged, gc.IsNil)
+                }
+        }
+}
+
+func (*metadataHelperSuite) TestReadWriteMetadata(c *gc.C) {
+        metadata := []*tools.ToolsMetadata{{
+                Release: "precise",
+                Version: "1.2.3",
+                Arch:    "amd64",
+                Path:    "path1",
+        }, {
+                Release: "raring",
+                Version: "1.2.3",
+                Arch:    "amd64",
+                Path:    "path2",
+        }}
+
+        stor, err := filestorage.NewFileStorageWriter(c.MkDir(), filestorage.UseDefaultTmpDir)
+        c.Assert(err, gc.IsNil)
+        out, err := tools.ReadMetadata(stor)
+        c.Assert(out, gc.HasLen, 0)
+        c.Assert(err, gc.IsNil) // non-existence is not an error
+        err = tools.WriteMetadata(stor, metadata)
+        c.Assert(err, gc.IsNil)
+        out, err = tools.ReadMetadata(stor)
+        for _, md := range out {
+                // FullPath is set by ReadMetadata.
+                c.Assert(md.FullPath, gc.Not(gc.Equals), "")
+                md.FullPath = ""
+        }
+        c.Assert(out, gc.DeepEquals, metadata)
+}