Agent config format 1.18

agent/agent.go

 // Copyright 2013 Canonical Ltd.
 // Licensed under the AGPLv3, see LICENCE file for details.
 
 package agent
 
 import (
+        "bytes"
         "fmt"
+        "io/ioutil"
+        "os"
         "path"
+        "path/filepath"
         "regexp"
+        "strings"
         "sync"
 
         "github.com/errgo/errgo"
         "github.com/loggo/loggo"
 
         "launchpad.net/juju-core/errors"
         "launchpad.net/juju-core/state"
         "launchpad.net/juju-core/state/api"
         "launchpad.net/juju-core/state/api/params"
         "launchpad.net/juju-core/utils"
         "launchpad.net/juju-core/version"
 )
 
 var logger = loggo.GetLogger("juju.agent")
 
+// DefaultLogDir defines the default log directory for juju agents.
+const DefaultLogDir = "/var/log/juju"
+
 const (
         LxcBridge        = "LXC_BRIDGE"
         ProviderType     = "PROVIDER_TYPE"
         ContainerType    = "CONTAINER_TYPE"
         Namespace        = "NAMESPACE"
         StorageDir       = "STORAGE_DIR"
         StorageAddr      = "STORAGE_ADDR"
         AgentServiceName = "AGENT_SERVICE_NAME"
         MongoServiceName = "MONGO_SERVICE_NAME"
-        BootstrapJobs    = "BOOTSTRAP_JOBS"
 )
 
 // The Config interface is the sole way that the agent gets access to the
 // configuration information for the machine and unit agents.  There should
 // only be one instance of a config object for any given agent, and this
 // interface is passed between multiple go routines.  The mutable methods are
 // protected by a mutex, and it is expected that the caller doesn't modify any
 // slice that may be returned.
 //
 // NOTE: should new mutating methods be added to this interface, consideration
 // is needed around the synchronisation as a single instance is used in
 // multiple go routines.
 type Config interface {
         // DataDir returns the data directory. Each agent has a subdirectory
         // containing the configuration files.
         DataDir() string
 
+        // LogDir returns the log directory. All logs from all agents on
+        // the machine are written to this directory.
+        LogDir() string
+
+        // Jobs returns a list of MachineJobs that need to run.
+        Jobs() []params.MachineJob
+
         // Tag returns the tag of the entity on whose behalf the state connection
         // will be made.
         Tag() string
 
         // Dir returns the agent's directory.
         Dir() string
 
         // Nonce returns the nonce saved when the machine was provisioned
         // TODO: make this one of the key/value pairs.
         Nonce() string
@@ skipping 61 matching lines @@
 // the values map.  Retrieving and setting values here are protected by the
 // mutex.  New mutating methods should also be synchronized using this mutex.
 var configMutex sync.Mutex
 
 type connectionDetails struct {
         addresses []string
         password  string
 }
 
 type configInternal struct {
+        configFilePath    string
         dataDir           string
+        logDir            string
         tag               string
+        nonce             string
+        jobs              []params.MachineJob
         upgradedToVersion version.Number
-        nonce             string
         caCert            []byte
         stateDetails      *connectionDetails
         apiDetails        *connectionDetails
         oldPassword       string
         stateServerCert   []byte
         stateServerKey    []byte
         apiPort           int
         values            map[string]string
 }
 
 type AgentConfigParams struct {
         DataDir           string
+        LogDir            string
+        Jobs              []params.MachineJob
+        UpgradedToVersion version.Number
         Tag               string
-        UpgradedToVersion version.Number
         Password          string
         Nonce             string
         StateAddresses    []string
         APIAddresses      []string
         CACert            []byte
         Values            map[string]string
 }
 
 // NewAgentConfig returns a new config object suitable for use for a
 // machine or unit agent.
 func NewAgentConfig(configParams AgentConfigParams) (Config, error) {
         if configParams.DataDir == "" {
                 return nil, errgo.Trace(requiredError("data directory"))
         }
+        logDir := DefaultLogDir
+        if configParams.LogDir != "" {
+                logDir = configParams.LogDir
+        }
         if configParams.Tag == "" {
                 return nil, errgo.Trace(requiredError("entity tag"))
         }
         if configParams.UpgradedToVersion == version.Zero {
                 return nil, errgo.Trace(requiredError("upgradedToVersion"))
         }
         if configParams.Password == "" {
                 return nil, errgo.Trace(requiredError("password"))
         }
         if configParams.CACert == nil {
                 return nil, errgo.Trace(requiredError("CA certificate"))
         }
         // Note that the password parts of the state and api information are
         // blank.  This is by design.
         config := &configInternal{
+                logDir:            logDir,
                 dataDir:           configParams.DataDir,
+                jobs:              configParams.Jobs,
+                upgradedToVersion: configParams.UpgradedToVersion,
                 tag:               configParams.Tag,
-                upgradedToVersion: configParams.UpgradedToVersion,
                 nonce:             configParams.Nonce,
                 caCert:            configParams.CACert,
                 oldPassword:       configParams.Password,
                 values:            configParams.Values,
         }
         if len(configParams.StateAddresses) > 0 {
                 config.stateDetails = &connectionDetails{
                         addresses: configParams.StateAddresses,
                 }
         }
         if len(configParams.APIAddresses) > 0 {
                 config.apiDetails = &connectionDetails{
                         addresses: configParams.APIAddresses,
                 }
         }
         if err := config.check(); err != nil {
                 return nil, err
         }
         if config.values == nil {
                 config.values = make(map[string]string)
         }
+        config.configFilePath = ConfigPath(config.dataDir, config.tag)
         return config, nil
 }
 
 type StateMachineConfigParams struct {
         AgentConfigParams
         StateServerCert []byte
         StateServerKey  []byte
         StatePort       int
         APIPort         int
 }
@@ skipping 13 matching lines @@
         }
         config := config0.(*configInternal)
         config.stateServerCert = configParams.StateServerCert
         config.stateServerKey = configParams.StateServerKey
         config.apiPort = configParams.APIPort
         return config, nil
 }
 
 // Dir returns the agent-specific data directory.
 func Dir(dataDir, agentName string) string {
-        return path.Join(dataDir, "agents", agentName)
+        return filepath.Join(dataDir, "agents", agentName)
 }
 
-// ReadConf reads configuration data for the given
-// entity from the given data directory.
-func ReadConf(dataDir, tag string) (Config, error) {
+// ConfigPath returns the full path to the agent config file.
+// NOTE: Delete this once all agents accept --config instead
+// of --data-dir - it won't be needed anymore.
+func ConfigPath(dataDir, agentName string) string {
+        return filepath.Join(Dir(dataDir, agentName), agentConfigFilename)
+}
+
+// ReadConf reads configuration data from the given location.
+func ReadConf(configFilePath string) (Config, error) {
         // Even though the ReadConf is done at the start of the agent loading, and
         // that this should not be called more than once by an agent, I feel that
         // not locking the mutex that is used to protect writes is wrong.
         configMutex.Lock()
         defer configMutex.Unlock()
-        dir := Dir(dataDir, tag)
-        format, err := readFormat(dir)
+        var (
+                format formatter
+                config *configInternal
+                err    error

[rog, 2014/03/06 15:14:07]

d

[dimitern, 2014/03/06 16:10:33]

Done.

+        )
+        configData, err := ioutil.ReadFile(configFilePath)
+        if err != nil {
+                return nil, fmt.Errorf("cannot read agent config %q: %v", configFilePath, err)
+        }
+
+        // Try to read the legacy format file.
+        dir := filepath.Dir(configFilePath)
+        legacyFormatPath := filepath.Join(dir, legacyFormatFilename)
+        formatBytes, err := ioutil.ReadFile(legacyFormatPath)

[rog, 2014/03/06 15:14:07]

if err != nil && !os.IsNotExist(err) {
    return nil, fmt.Errorf("cannot read format file: %v", err)
}

?

[dimitern, 2014/03/06 16:10:33]

Done.

+        formatData := string(formatBytes)
+        if err == nil {
+                // It exists, so unmarshal with a legacy formatter.
+                // Drop the format prefix to leave the version only.
+                if !strings.HasPrefix(formatData, legacyFormatPrefix) {
+                        return nil, fmt.Errorf("malformed agent config format: %q", formatData)

[rog, 2014/03/06 15:14:07]

s/: //

[dimitern, 2014/03/06 16:10:33]

Done.

+                }
+                format, err = getFormatter(strings.TrimPrefix(formatData, legacyFormatPrefix))
+                if err != nil {
+                        return nil, err
+                }
+                config, err = format.unmarshal(configData)
+        } else {
+                // Does not exist, just parse the data.
+                format, config, err = parseConfigData(configData)
+        }
         if err != nil {
                 return nil, err
         }
-        logger.Debugf("Reading agent config, format: %s", format)
-        formatter, err := newFormatter(format)
-        if err != nil {
-                return nil, err
-        }
-        config, err := formatter.read(dir)
-        if err != nil {
-                return nil, err
-        }
-        config.dataDir = dataDir
-        if err := config.check(); err != nil {
-                return nil, err
-        }
-
+        logger.Debugf("read agent config format %q", format.version())

[rog, 2014/03/06 15:14:07]

s/config/config, /
?

[dimitern, 2014/03/06 16:10:33]

Done.

         if format != currentFormat {
-                // Migrate the config to the new format.
-                currentFormatter.migrate(config)
-                // Write the content out in the new format.
-                if err := currentFormatter.write(config); err != nil {
-                        logger.Errorf("cannot write the agent config in format %s: %v", currentFormat, err)
-                        return nil, err
+                // Migrate from a legacy format to the new one.
+                data, err := currentFormat.marshal(config)
+                if err != nil {
+                        return nil, fmt.Errorf("cannot marshal agent config: %v", err)
+                }
+                err = utils.AtomicWriteFile(configFilePath, data, 0600)

[rog, 2014/03/06 15:14:07]

This doesn't look quite right - we're not writing
the header here.

We should have a test for this.

I suggest we just call config.Write here instead (as well as
removing the legacy format file). Since Write locks the
mutex, probably make an internal write method that doesn't
lock, and call that from here and from Write.

[dimitern, 2014/03/06 16:10:33]

Done.

+                if err != nil {
+                        return nil, fmt.Errorf("cannot upgrade agent config %s to %s: %v", format.version(), currentFormat.version(), err)
+                }
+                logger.Debugf("migrated agent config from %s to %s", format.version(), currentFormat.version())
+                err = os.Remove(legacyFormatPath)
+                if err != nil && !os.IsNotExist(err) {
+                        return nil, fmt.Errorf("cannot remove legacy format file %q: %v", legacyFormatPath, err)
                 }
         }
-
+        config.configFilePath = configFilePath
         return config, nil
 }
 
 func requiredError(what string) error {
         return fmt.Errorf("%s not found in configuration", what)
 }
 
 func (c *configInternal) File(name string) string {
         return path.Join(c.Dir(), name)
 }
 
 func (c *configInternal) DataDir() string {
         return c.dataDir
 }
 
+func (c *configInternal) LogDir() string {
+        return c.logDir
+}
+
+func (c *configInternal) Jobs() []params.MachineJob {
+        return c.jobs
+}
+
 func (c *configInternal) Nonce() string {
         return c.nonce
 }
 
 func (c *configInternal) UpgradedToVersion() version.Number {
         return c.upgradedToVersion
 }
 
 func (c *configInternal) CACert() []byte {
         // Give the caller their own copy of the cert to avoid any possibility of
@@ skipping 90 matching lines @@
                 other.apiDetails = &apiDetails
         }
         logger.Debugf("writing configuration file")
         if err := other.Write(); err != nil {
                 return "", err
         }
         *c = other
         return newPassword, nil
 }
 
+func (c *configInternal) fileContents() ([]byte, error) {
+        data, err := currentFormat.marshal(c)
+        if err != nil {
+                return nil, err
+        }
+        var buf bytes.Buffer
+        fmt.Fprintf(&buf, "%s%s\n", formatPrefix, currentFormat.version())
+        buf.Write(data)
+        return buf.Bytes(), nil
+}
+
 func (c *configInternal) Write() error {
         // Lock is taken prior to generating any content to write.
         configMutex.Lock()
         defer configMutex.Unlock()
-        return currentFormatter.write(c)
+        data, err := c.fileContents()
+        if err != nil {
+                return err
+        }
+        // Make sure the config dir gets created.
+        configDir := filepath.Dir(c.configFilePath)
+        if err := os.MkdirAll(configDir, 0755); err != nil {
+                return fmt.Errorf("cannot create agent config dir %q: %v", configDir, err)
+        }
+        return utils.AtomicWriteFile(c.configFilePath, data, 0600)
 }
 
 func (c *configInternal) WriteUpgradedToVersion(newVersion version.Number) error {
         originalVersion := c.upgradedToVersion
         c.upgradedToVersion = newVersion
         err := c.Write()
         if err != nil {
                 // We don't want to retain the new version if there's been an error writing the file.
                 c.upgradedToVersion = originalVersion
         }
         return err
 }
 
 func (c *configInternal) WriteCommands() ([]string, error) {
-        return currentFormatter.writeCommands(c)
+        data, err := c.fileContents()
+        if err != nil {
+                return nil, err
+        }
+        commands := []string{"mkdir -p " + utils.ShQuote(c.Dir())}
+        commands = append(commands, writeFileCommands(c.File(agentConfigFilename), data, 0600)...)
+        return commands, nil
 }
 
 func (c *configInternal) OpenAPI(dialOpts api.DialOpts) (st *api.State, newPassword string, err error) {
         info := api.Info{
                 Addrs:    c.apiDetails.addresses,
                 Password: c.apiDetails.password,
                 CACert:   c.caCert,
                 Tag:      c.tag,
                 Nonce:    c.nonce,
         }
@@ skipping 40 matching lines @@
                 }
                 // TODO(rog) remove this fallback behaviour when
                 // all initial connections are via the API.
                 if !errors.IsUnauthorizedError(err) {
                         return nil, err
                 }
         }
         info.Password = c.oldPassword
         return state.Open(&info, state.DefaultDialOpts(), policy)
 }