Reduce unnecessary s3 timeouts

provider/ec2/storage.go

 // Copyright 2012, 2013 Canonical Ltd.
 // Licensed under the AGPLv3, see LICENCE file for details.
 
 package ec2
 
 import (
         "fmt"
         "io"
+        "net"
         "sync"
         "time"
 
         "launchpad.net/goamz/s3"
 
-        "launchpad.net/juju-core/environs"
+        "launchpad.net/juju-core/environs/storage"
         "launchpad.net/juju-core/errors"
         "launchpad.net/juju-core/utils"
-        "net"

[thumper, 2013/09/18 02:50:55]

this should go with the top block of standard libraries

[wallyworld, 2013/09/18 07:29:16]

Done.

 )
 
 func init() {
         // We will decide when to retry and under what circumstances, not s3.
         // Sometimes it is expected a file may not exist and we don't want s3
         // to hold things up by unilaterally deciding to retry for no good reason.
         s3.RetryAttempts(false)
 }
 
-func NewStorage(bucket *s3.Bucket) environs.Storage {
-        return &storage{bucket: bucket}
-}
-
-// storage implements environs.Storage on
+func NewStorage(bucket *s3.Bucket) storage.Storage {
+        return &ec2storage{bucket: bucket}
+}
+
+// ec2storage implements storage.Storage on
 // an ec2.bucket.
-type storage struct {
+type ec2storage struct {
         sync.Mutex
         madeBucket bool
         bucket     *s3.Bucket
 }
 
 // makeBucket makes the environent's control bucket, the
 // place where bootstrap information and deployed charms
 // are stored. To avoid two round trips on every PUT operation,
 // we do this only once for each environ.
-func (s *storage) makeBucket() error {
+func (s *ec2storage) makeBucket() error {
         s.Lock()
         defer s.Unlock()
         if s.madeBucket {
                 return nil
         }
         // PutBucket always return a 200 if we recreate an existing bucket for the
         // original s3.amazonaws.com endpoint. For all other endpoints PutBucket
         // returns 409 with a known subcode.
         if err := s.bucket.PutBucket(s3.Private); err != nil && s3ErrCode(err) != "BucketAlreadyOwnedByYou" {
                 return err
         }
 
         s.madeBucket = true
         return nil
 }
 
-func (s *storage) Put(file string, r io.Reader, length int64) error {
+func (s *ec2storage) Put(file string, r io.Reader, length int64) error {
         if err := s.makeBucket(); err != nil {
                 return fmt.Errorf("cannot make S3 control bucket: %v", err)
         }
         err := s.bucket.PutReader(file, r, length, "binary/octet-stream", s3.Private)
         if err != nil {
                 return fmt.Errorf("cannot write file %q to control bucket: %v", file, err)
         }
         return nil
 }
 
-func (s *storage) Get(file string) (r io.ReadCloser, err error) {
+func (s *ec2storage) Get(file string) (r io.ReadCloser, err error) {
         r, err = s.bucket.GetReader(file)
         return r, maybeNotFound(err)
 }
 
-func (s *storage) URL(name string) (string, error) {
+func (s *ec2storage) URL(name string) (string, error) {
         // 10 years should be good enough.
         return s.bucket.SignedURL(name, time.Now().AddDate(10, 0, 0)), nil
 }
 
 var storageAttempt = utils.AttemptStrategy{
         Total: 5 * time.Second,
         Delay: 200 * time.Millisecond,
 }
 
 // ConsistencyStrategy is specified in the StorageReader interface.
-func (s *storage) DefaultConsistencyStrategy() utils.AttemptStrategy {
+func (s *ec2storage) DefaultConsistencyStrategy() utils.AttemptStrategy {
         return storageAttempt
 }
 
 // ShouldRetry is specified in the StorageReader interface.
-func (s *storage) ShouldRetry(err error) bool {
+func (s *ec2storage) ShouldRetry(err error) bool {
         if err == nil {
                 return false
         }
         switch err {
         case io.ErrUnexpectedEOF, io.EOF:
                 return true
         }
         if s3ErrorStatusCode(err) == 404 {
                 return true
         }
@@ skipping 24 matching lines @@
 }
 
 // s3ErrCode returns the text status code of the S3 error code.
 func s3ErrCode(err error) string {
         if err, ok := err.(*s3.Error); ok {
                 return err.Code
         }
         return ""
 }
 
-func (s *storage) Remove(file string) error {
+func (s *ec2storage) Remove(file string) error {
         err := s.bucket.Del(file)
         // If we can't delete the object because the bucket doesn't
         // exist, then we don't care.
         if s3ErrorStatusCode(err) == 404 {
                 return nil
         }
         return err
 }
 
-func (s *storage) List(prefix string) ([]string, error) {
+func (s *ec2storage) List(prefix string) ([]string, error) {
         // TODO cope with more than 1000 objects in the bucket.
         resp, err := s.bucket.List(prefix, "", "", 0)
         if err != nil {
                 // If the bucket is not found, it's not an error
                 // because it's only created when the first
                 // file is put.
                 if s3ErrorStatusCode(err) == 404 {
                         return nil, nil
                 }
                 return nil, err
         }
         var names []string
         for _, key := range resp.Contents {
                 names = append(names, key.Key)
         }
         return names, nil
 }
 
-func (s *storage) RemoveAll() error {
-        names, err := environs.DefaultList(s, "")
+func (s *ec2storage) RemoveAll() error {
+        names, err := storage.ListWithDefaultRetry(s, "")
         if err != nil {
                 return err
         }
         // Remove all the objects in parallel to minimize round-trips.
         // If we're in danger of having hundreds of objects,
         // we'll want to change this to limit the number
         // of concurrent operations.
         var wg sync.WaitGroup
         wg.Add(len(names))
         errc := make(chan error, len(names))
@@ skipping 19 matching lines @@
         // might have succeeded even if we get an error.
         s.madeBucket = false
         err = deleteBucket(s)
         err = s.bucket.DelBucket()
         if s3ErrorStatusCode(err) == 404 {
                 return nil
         }
         return err
 }
 
-func deleteBucket(s *storage) (err error) {
+func deleteBucket(s *ec2storage) (err error) {
         for a := s.DefaultConsistencyStrategy().Start(); a.Next(); {
                 err = s.bucket.DelBucket()
                 if err == nil || !s.ShouldRetry(err) {
                         break
                 }
         }
         return err
 }
 
 func maybeNotFound(err error) error {
         if err != nil && s3ErrorStatusCode(err) == 404 {
                 return errors.NewNotFoundError(err, "")
         }
         return err
 }