// Set takes the object and commits it to the database.
func (c *Client) Set(obj db.Entity) error {
if err := obj.Validate(); err != nil {
return err
}
if obj.Hooks().PreSet != nil {
if err := obj.Hooks().PreSet(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
content, err := jsonio.Write(obj)
if err != nil {
return err
}
path, err := obj.Path()
if err != nil {
return err
}
if _, err := c.client.Set(context.Background(), c.qualified(path), string(content), nil); err != nil {
return errors.EtcdToErrored(err)
}
if obj.Hooks().PostSet != nil {
if err := obj.Hooks().PostSet(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
return nil
}
// Get retrieves the item from etcd's key/value store and then populates obj with its data.
func (c *Client) Get(obj db.Entity) error {
if obj.Hooks().PreGet != nil {
if err := obj.Hooks().PreGet(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
path, err := obj.Path()
if err != nil {
return err
}
resp, err := c.client.Get(context.Background(), c.qualified(path), nil)
if err != nil {
return errors.EtcdToErrored(err)
}
if err := jsonio.Read(obj, []byte(resp.Node.Value)); err != nil {
return err
}
if err := obj.SetKey(c.trimPath(resp.Node.Key)); err != nil {
return err
}
if obj.Hooks().PostGet != nil {
if err := obj.Hooks().PostGet(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
return obj.Validate()
}
// GetVolume returns the Volume for a given volume.
func (c *Client) GetVolume(policy, name string) (*Volume, error) {
// FIXME make this take a single string and not a split one
resp, err := c.etcdClient.Get(context.Background(), c.volume(policy, name, "create"), nil)
if err != nil {
return nil, errors.EtcdToErrored(err)
}
ret := &Volume{}
if err := json.Unmarshal([]byte(resp.Node.Value), ret); err != nil {
return nil, err
}
if err := ret.Validate(); err != nil {
return nil, err
}
runtime, err := c.GetVolumeRuntime(policy, name)
if err != nil {
return nil, err
}
ret.RuntimeOptions = runtime
return ret, nil
}
// PublishUseWithTTL pushes the use to etcd, with a TTL that expires the record
// if it has not been updated within that time.
func (c *Client) PublishUseWithTTL(ut UseLocker, ttl time.Duration) error {
content, err := json.Marshal(ut)
if err != nil {
return err
}
if ttl < 0 {
err := errored.Errorf("TTL was less than 0 for locker %#v!!!! This should not happen!", ut)
logrus.Error(err)
return err
}
logrus.Debugf("Publishing use with TTL %v: %#v", ttl, ut)
value := string(content)
// attempt to set the lock. If the lock cannot be set and it is is empty, attempt to set it now.
_, err = c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{TTL: ttl, PrevValue: value})
if err != nil {
if er, ok := errors.EtcdToErrored(err).(*errored.Error); ok && er.Contains(errors.NotExists) {
_, err := c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{TTL: ttl, PrevExist: client.PrevNoExist})
if err != nil {
return errors.PublishMount.Combine(err)
}
} else {
return errors.PublishMount.Combine(err)
}
}
return nil
}
// PublishPolicy publishes policy intent to the configuration store.
func (c *Client) PublishPolicy(name string, cfg *Policy) error {
cfg.Name = name
if err := cfg.Validate(); err != nil {
return err
}
value, err := json.Marshal(cfg)
if err != nil {
return err
}
// NOTE: The creation of the policy revision entry and the actual publishing of the policy
// should be wrapped in a transaction so they either both succeed or both fail, but
// etcd2 doesn't support transactions (etcd3 does/will).
//
// For now, we create the revision entry first and then publish the policy. It's
// better to have an entry for a policy revision that was never actually published
// than to have a policy published which has no revision recorded for it.
if err := c.CreatePolicyRevision(name, string(value)); err != nil {
return err
}
// create the volume directory for the policy so that files can be written there.
// for example: /volplugin/policies/policy1 will create
// /volplugin/volumes/policy1 so that a volume of policy1/test can be created
// at /volplugin/volumes/policy1/test
c.etcdClient.Set(context.Background(), c.prefixed(rootVolume, name), "", &client.SetOptions{Dir: true})
if _, err := c.etcdClient.Set(context.Background(), c.policy(name), string(value), &client.SetOptions{PrevExist: client.PrevIgnore}); err != nil {
return errors.EtcdToErrored(err)
}
return nil
}
// Delete removes the object from the store.
func (c *Client) Delete(obj db.Entity) error {
return helpers.WrapDelete(c, obj, func(path string) error {
if _, err := c.client.Delete(context.Background(), c.qualified(path), nil); err != nil {
return errors.EtcdToErrored(err)
}
return nil
})
}
// PublishUse pushes the use to etcd.
func (c *Client) PublishUse(ut UseLocker) error {
content, err := json.Marshal(ut)
if err != nil {
return err
}
_, err = c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{PrevExist: client.PrevNoExist})
if _, ok := err.(client.Error); ok && err.(client.Error).Code == client.ErrorCodeNodeExist {
if ut.MayExist() {
_, err := c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{PrevExist: client.PrevExist, PrevValue: string(content)})
return errors.EtcdToErrored(err)
}
return errors.Exists.Combine(err)
}
logrus.Debugf("Publishing use: (error: %v) %#v", err, ut)
return errors.EtcdToErrored(err)
}
// GetPolicyRevision returns a single revision for a given policy.
func (c *Client) GetPolicyRevision(name, revision string) (string, error) {
keyspace := c.policyArchiveEntry(name, revision)
resp, err := c.etcdClient.Get(context.Background(), keyspace, &client.GetOptions{Sort: false, Recursive: false, Quorum: true})
if err != nil {
return "", errors.EtcdToErrored(err)
}
return resp.Node.Value, nil
}
// GetVolumeRuntime retrieves only the runtime parameters for the volume.
func (c *Client) GetVolumeRuntime(policy, name string) (RuntimeOptions, error) {
runtime := RuntimeOptions{}
resp, err := c.etcdClient.Get(context.Background(), c.volume(policy, name, "runtime"), nil)
if err != nil {
return runtime, errors.EtcdToErrored(err)
}
return runtime, json.Unmarshal([]byte(resp.Node.Value), &runtime)
}
// Get retrieves the item from etcd's key/value store and then populates obj with its data.
func (c *Client) Get(obj db.Entity) error {
return helpers.WrapGet(c, obj, func(path string) (string, []byte, error) {
resp, err := c.client.Get(context.Background(), c.qualified(path), nil)
if err != nil {
return "", nil, errors.EtcdToErrored(err)
}
return resp.Node.Key, []byte(resp.Node.Value), nil
})
}
// CreatePolicyRevision creates an revision entry in a policy's history.
func (c *Client) CreatePolicyRevision(name string, policy string) error {
timestamp := fmt.Sprint(time.Now().Unix())
key := c.policyArchiveEntry(name, timestamp)
_, err := c.etcdClient.Set(context.Background(), key, policy, nil)
if err != nil {
return errors.EtcdToErrored(err)
}
return nil
}
// ListVolumes returns a map of volume name -> Volume.
func (c *Client) ListVolumes(policy string) (map[string]*Volume, error) {
policyPath := c.prefixed(rootVolume, policy)
resp, err := c.etcdClient.Get(context.Background(), policyPath, &client.GetOptions{Recursive: true, Sort: true})
if err != nil {
return nil, errors.EtcdToErrored(err)
}
configs := map[string]*Volume{}
for _, node := range resp.Node.Nodes {
if len(node.Nodes) > 0 {
node = node.Nodes[0]
key := strings.TrimPrefix(node.Key, policyPath)
if !node.Dir && strings.HasSuffix(node.Key, "/create") {
key = strings.TrimSuffix(key, "/create")
config, ok := configs[key[1:]]
if !ok {
config = new(Volume)
}
if err := json.Unmarshal([]byte(node.Value), config); err != nil {
return nil, err
}
// trim leading slash
configs[key[1:]] = config
}
if !node.Dir && strings.HasSuffix(node.Key, "/runtime") {
key = strings.TrimSuffix(key, "/create")
config, ok := configs[key[1:]]
if !ok {
config = new(Volume)
}
if err := json.Unmarshal([]byte(node.Value), &config.RuntimeOptions); err != nil {
return nil, err
}
// trim leading slash
configs[key[1:]] = config
}
}
}
for _, config := range configs {
if _, err := config.CreateOptions.ActualSize(); err != nil {
return nil, err
}
}
return configs, nil
}
// ListAllVolumes returns an array with all the named policies and volumes the
// apiserver knows about. Volumes have syntax: policy/volumeName which will be
// reflected in the returned string.
func (c *Client) ListAllVolumes() ([]string, error) {
resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootVolume), &client.GetOptions{Recursive: true, Sort: true})
if err != nil {
if er, ok := errors.EtcdToErrored(err).(*errored.Error); ok && er.Contains(errors.NotExists) {
return []string{}, nil
}
return nil, errors.EtcdToErrored(err)
}
ret := []string{}
for _, node := range resp.Node.Nodes {
for _, innerNode := range node.Nodes {
ret = append(ret, path.Join(path.Base(node.Key), path.Base(innerNode.Key)))
}
}
return ret, nil
}
// GetUse retrieves the UseMount for the given volume name.
func (c *Client) GetUse(ut UseLocker, vc *Volume) error {
resp, err := c.etcdClient.Get(context.Background(), c.use(ut.Type(), vc.String()), nil)
if err != nil {
return errors.EtcdToErrored(err)
}
if err := json.Unmarshal([]byte(resp.Node.Value), ut); err != nil {
return err
}
return nil
}
// PublishGlobal publishes the global configuration.
func (tlc *Client) PublishGlobal(g *Global) error {
gcPath := tlc.prefixed("global-config")
value, err := json.Marshal(g.Canonical())
if err != nil {
return err
}
if _, err := tlc.etcdClient.Set(context.Background(), gcPath, string(value), &client.SetOptions{PrevExist: client.PrevIgnore}); err != nil {
return errors.EtcdToErrored(err)
}
return nil
}
// Delete removes the object from the store.
func (c *Client) Delete(obj db.Entity) error {
if obj.Hooks().PreDelete != nil {
if err := obj.Hooks().PreDelete(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
path, err := obj.Path()
if err != nil {
return err
}
if _, err := c.client.Delete(context.Background(), c.qualified(path), nil); err != nil {
return errors.EtcdToErrored(err)
}
if obj.Hooks().PostDelete != nil {
if err := obj.Hooks().PostDelete(c, obj); err != nil {
return errors.EtcdToErrored(err)
}
}
return nil
}
// RemoveUse will remove a user from etcd. Does not fail if the user does
// not exist.
func (c *Client) RemoveUse(ut UseLocker, force bool) error {
content, err := json.Marshal(ut)
if err != nil {
return err
}
logrus.Debugf("Removing Use Lock: %#v", ut)
opts := &client.DeleteOptions{PrevValue: string(content)}
if force {
opts = nil
}
_, err = c.etcdClient.Delete(context.Background(), c.use(ut.Type(), ut.GetVolume()), opts)
return errors.EtcdToErrored(err)
}
// PublishVolumeRuntime publishes the runtime parameters for each volume.
func (c *Client) PublishVolumeRuntime(vo *Volume, ro RuntimeOptions) error {
if err := ro.ValidateJSON(); err != nil {
return errors.ErrJSONValidation.Combine(err)
}
content, err := json.Marshal(ro)
if err != nil {
return err
}
c.etcdClient.Set(context.Background(), c.prefixed(rootVolume, vo.PolicyName, vo.VolumeName), "", &client.SetOptions{Dir: true})
if _, err := c.etcdClient.Set(context.Background(), c.volume(vo.PolicyName, vo.VolumeName, "runtime"), string(content), nil); err != nil {
return errors.EtcdToErrored(err)
}
return nil
}
// CurrentSchemaVersion returns the version of the last migration which was successfully run.
// If no previous migrations have been run, the schema version is 0.
// If there's any error besides "key doesn't exist", execution is aborted.
func (e *Engine) CurrentSchemaVersion() int64 {
resp, err := e.etcdClient.Get(context.Background(), path.Join(e.prefix, backend.SchemaVersionKey), nil)
if err != nil {
if err := errors.EtcdToErrored(err); err != nil && err == errors.NotExists {
return 0 // no key = schema version 0
}
logrus.Fatalf("Unexpected error when looking up schema version: %v\n", err)
}
i, err := strconv.Atoi(resp.Node.Value)
if err != nil {
logrus.Fatalf("Got back unexpected schema version data: %v\n", resp.Node.Value)
}
return int64(i)
}
// ListPolicies provides an array of strings corresponding to the name of each
// policy.
func (c *Client) ListPolicies() ([]Policy, error) {
resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootPolicy), &client.GetOptions{Recursive: true, Sort: true})
if err != nil {
return nil, errors.EtcdToErrored(err)
}
policies := []Policy{}
for _, node := range resp.Node.Nodes {
policy := Policy{}
if err := json.Unmarshal([]byte(node.Value), &policy); err != nil {
return nil, err
}
policies = append(policies, policy)
}
return policies, nil
}
// ListPolicyRevisions returns a list of all the revisions for a given policy.
func (c *Client) ListPolicyRevisions(name string) ([]string, error) {
keyspace := c.policyArchive(name)
resp, err := c.etcdClient.Get(context.Background(), keyspace, &client.GetOptions{Sort: true, Recursive: false, Quorum: true})
if err != nil {
return nil, errors.EtcdToErrored(err)
}
ret := []string{}
for _, node := range resp.Node.Nodes {
_, revision := path.Split(node.Key)
ret = append(ret, revision)
}
return ret, nil
}
// ListUses lists the items in use.
func (c *Client) ListUses(typ string) ([]string, error) {
resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootUse, typ), &client.GetOptions{Sort: true, Recursive: true})
if err != nil {
return nil, errors.EtcdToErrored(err)
}
ret := []string{}
for _, node := range resp.Node.Nodes {
for _, inner := range node.Nodes {
key := path.Join(strings.TrimPrefix(inner.Key, c.prefixed(rootUse, typ)))
// trim leading slash
key = key[1:]
ret = append(ret, key)
}
}
return ret, nil
}
// GetPolicy retrieves a policy from the configuration store.
func (c *Client) GetPolicy(name string) (*Policy, error) {
if name == "" {
return nil, errored.Errorf("Policy invalid: empty string for name")
}
resp, err := c.etcdClient.Get(context.Background(), c.policy(name), nil)
if err != nil {
return nil, errors.EtcdToErrored(err)
}
tc := NewPolicy()
if err := json.Unmarshal([]byte(resp.Node.Value), tc); err != nil {
return nil, err
}
tc.Name = name
err = tc.Validate()
return tc, err
}
func (dc *DaemonConfig) loop() {
for {
time.Sleep(time.Second)
// XXX this copy is so we can free the mutex quickly for more additions
volumeCopy := map[string]*config.Volume{}
volumeMutex.Lock()
for volume, val := range volumes {
logrus.Debugf("Adding volume %q for processing", volume)
val2 := *val
volumeCopy[volume] = &val2
}
volumeMutex.Unlock()
for volume, val := range volumeCopy {
if val.RuntimeOptions.UseSnapshots {
freq, err := time.ParseDuration(val.RuntimeOptions.Snapshot.Frequency)
if err != nil {
logrus.Errorf("Volume %q has an invalid frequency. Skipping snapshot.", volume)
}
if time.Now().Unix()%int64(freq.Seconds()) == 0 {
var isUsed bool
var err error
if isUsed, err = dc.Config.IsVolumeInUse(val, dc.Global); err != nil {
logrus.Errorf("etcd error: %s", errors.EtcdToErrored(err)) // some issue with "etcd GET"; we should not hit this case
}
go func(val *config.Volume, isUsed bool) {
// XXX we still want to prune snapshots even if the volume is not in use.
if isUsed {
dc.createSnapshot(val)
}
dc.pruneSnapshots(val)
}(val, isUsed)
}
}
}
}
}
// NewClient creates a new Client with connections to etcd already established.
func NewClient(hosts []string, prefix string) (*Client, error) {
ec, err := client.New(client.Config{Endpoints: hosts})
if err != nil {
return nil, err
}
c := &Client{
client: client.NewKeysAPI(ec),
prefix: prefix,
watchers: map[string]chan struct{}{},
}
if _, err := c.client.Set(context.Background(), c.prefix, "", &client.SetOptions{Dir: true, PrevExist: client.PrevNoExist}); err != nil {
if err != nil {
er, ok := errors.EtcdToErrored(err).(*errored.Error)
if !ok || !er.Contains(errors.Exists) {
return nil, errored.New("Initial setup").Combine(err)
}
}
}
return c, nil
}
// RemoveTakeSnapshot removes a reference to a taken snapshot, intended to be used by volsupervisor
func (c *Client) RemoveTakeSnapshot(name string) error {
_, err := c.etcdClient.Delete(context.Background(), c.prefixed(rootSnapshots, name), nil)
return errors.EtcdToErrored(err)
}
// RemoveVolume removes a volume from configuration.
func (c *Client) RemoveVolume(policy, name string) error {
logrus.Debugf("Removing volume %s/%s from database", policy, name)
_, err := c.etcdClient.Delete(context.Background(), c.prefixed(rootVolume, policy, name), &client.DeleteOptions{Recursive: true})
return errors.EtcdToErrored(err)
}
// DumpTarball dumps all the keys under the current etcd prefix into a
// gzip'd tarball'd directory-based representation of the namespace.
func (c *Client) DumpTarball() (string, error) {
resp, err := c.etcdClient.Get(context.Background(), c.prefix, &client.GetOptions{Sort: true, Recursive: true, Quorum: true})
if err != nil {
return "", errored.Errorf(`Failed to recursively GET "%s" namespace from etcd`, c.prefix).Combine(errors.EtcdToErrored(err))
}
now := time.Now()
// tar hangs during unpacking if the base directory has colons in it
// unless --force-local is specified, so use the simpler "%Y%m%d-%H%M%S".
niceTimeFormat := fmt.Sprintf("%d%02d%02d-%02d%02d%02d",
now.Year(), now.Month(), now.Day(),
now.Hour(), now.Minute(), now.Second())
file, err := ioutil.TempFile("", "etcd_dump_"+niceTimeFormat+"_")
if err != nil {
return "", errored.Errorf("Failed to create tempfile").Combine(err)
}
defer file.Close()
// create a gzipped, tarball writer which cleans up after itself
gzipWriter := gzip.NewWriter(file)
defer gzipWriter.Close()
tarWriter := tar.NewWriter(gzipWriter)
defer tarWriter.Close()
// ensure that the tarball extracts to a folder with the same name as the tarball
baseDirectory := filepath.Base(file.Name())
err = addNodeToTarball(resp.Node, tarWriter, baseDirectory)
if err != nil {
return "", err
}
// give the file a more fitting name
newFilename := file.Name() + ".tar.gz"
err = os.Rename(file.Name(), newFilename)
if err != nil {
return "", err
}
return newFilename, nil
}
// DeletePolicy removes a policy from the configuration store.
func (c *Client) DeletePolicy(name string) error {
_, err := c.etcdClient.Delete(context.Background(), c.policy(name), nil)
return errors.EtcdToErrored(err)
}