Example #1
0
// Set takes the object and commits it to the database.
func (c *Client) Set(obj db.Entity) error {
	if err := obj.Validate(); err != nil {
		return err
	}

	if obj.Hooks().PreSet != nil {
		if err := obj.Hooks().PreSet(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	content, err := jsonio.Write(obj)
	if err != nil {
		return err
	}

	path, err := obj.Path()
	if err != nil {
		return err
	}

	if _, err := c.client.Set(context.Background(), c.qualified(path), string(content), nil); err != nil {
		return errors.EtcdToErrored(err)
	}

	if obj.Hooks().PostSet != nil {
		if err := obj.Hooks().PostSet(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	return nil
}
Example #2
0
// Get retrieves the item from etcd's key/value store and then populates obj with its data.
func (c *Client) Get(obj db.Entity) error {
	if obj.Hooks().PreGet != nil {
		if err := obj.Hooks().PreGet(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	path, err := obj.Path()
	if err != nil {
		return err
	}

	resp, err := c.client.Get(context.Background(), c.qualified(path), nil)
	if err != nil {
		return errors.EtcdToErrored(err)
	}

	if err := jsonio.Read(obj, []byte(resp.Node.Value)); err != nil {
		return err
	}

	if err := obj.SetKey(c.trimPath(resp.Node.Key)); err != nil {
		return err
	}

	if obj.Hooks().PostGet != nil {
		if err := obj.Hooks().PostGet(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	return obj.Validate()
}
Example #3
0
// GetVolume returns the Volume for a given volume.
func (c *Client) GetVolume(policy, name string) (*Volume, error) {
	// FIXME make this take a single string and not a split one
	resp, err := c.etcdClient.Get(context.Background(), c.volume(policy, name, "create"), nil)
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	ret := &Volume{}

	if err := json.Unmarshal([]byte(resp.Node.Value), ret); err != nil {
		return nil, err
	}

	if err := ret.Validate(); err != nil {
		return nil, err
	}

	runtime, err := c.GetVolumeRuntime(policy, name)
	if err != nil {
		return nil, err
	}

	ret.RuntimeOptions = runtime

	return ret, nil
}
Example #4
0
// PublishUseWithTTL pushes the use to etcd, with a TTL that expires the record
// if it has not been updated within that time.
func (c *Client) PublishUseWithTTL(ut UseLocker, ttl time.Duration) error {
	content, err := json.Marshal(ut)
	if err != nil {
		return err
	}

	if ttl < 0 {
		err := errored.Errorf("TTL was less than 0 for locker %#v!!!! This should not happen!", ut)
		logrus.Error(err)
		return err
	}

	logrus.Debugf("Publishing use with TTL %v: %#v", ttl, ut)
	value := string(content)

	// attempt to set the lock. If the lock cannot be set and it is is empty, attempt to set it now.
	_, err = c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{TTL: ttl, PrevValue: value})
	if err != nil {
		if er, ok := errors.EtcdToErrored(err).(*errored.Error); ok && er.Contains(errors.NotExists) {
			_, err := c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{TTL: ttl, PrevExist: client.PrevNoExist})
			if err != nil {
				return errors.PublishMount.Combine(err)
			}
		} else {
			return errors.PublishMount.Combine(err)
		}
	}

	return nil
}
Example #5
0
// PublishPolicy publishes policy intent to the configuration store.
func (c *Client) PublishPolicy(name string, cfg *Policy) error {
	cfg.Name = name

	if err := cfg.Validate(); err != nil {
		return err
	}

	value, err := json.Marshal(cfg)
	if err != nil {
		return err
	}

	// NOTE: The creation of the policy revision entry and the actual publishing of the policy
	//       should be wrapped in a transaction so they either both succeed or both fail, but
	//       etcd2 doesn't support transactions (etcd3 does/will).
	//
	//       For now, we create the revision entry first and then publish the policy.  It's
	//       better to have an entry for a policy revision that was never actually published
	//       than to have a policy published which has no revision recorded for it.
	if err := c.CreatePolicyRevision(name, string(value)); err != nil {
		return err
	}

	// create the volume directory for the policy so that files can be written there.
	// for example: /volplugin/policies/policy1 will create
	// /volplugin/volumes/policy1 so that a volume of policy1/test can be created
	// at /volplugin/volumes/policy1/test
	c.etcdClient.Set(context.Background(), c.prefixed(rootVolume, name), "", &client.SetOptions{Dir: true})

	if _, err := c.etcdClient.Set(context.Background(), c.policy(name), string(value), &client.SetOptions{PrevExist: client.PrevIgnore}); err != nil {
		return errors.EtcdToErrored(err)
	}

	return nil
}
Example #6
0
// Delete removes the object from the store.
func (c *Client) Delete(obj db.Entity) error {
	return helpers.WrapDelete(c, obj, func(path string) error {
		if _, err := c.client.Delete(context.Background(), c.qualified(path), nil); err != nil {
			return errors.EtcdToErrored(err)
		}

		return nil
	})
}
Example #7
0
// PublishUse pushes the use to etcd.
func (c *Client) PublishUse(ut UseLocker) error {
	content, err := json.Marshal(ut)
	if err != nil {
		return err
	}

	_, err = c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{PrevExist: client.PrevNoExist})
	if _, ok := err.(client.Error); ok && err.(client.Error).Code == client.ErrorCodeNodeExist {
		if ut.MayExist() {
			_, err := c.etcdClient.Set(context.Background(), c.use(ut.Type(), ut.GetVolume()), string(content), &client.SetOptions{PrevExist: client.PrevExist, PrevValue: string(content)})
			return errors.EtcdToErrored(err)
		}
		return errors.Exists.Combine(err)
	}

	logrus.Debugf("Publishing use: (error: %v) %#v", err, ut)
	return errors.EtcdToErrored(err)
}
Example #8
0
// GetPolicyRevision returns a single revision for a given policy.
func (c *Client) GetPolicyRevision(name, revision string) (string, error) {
	keyspace := c.policyArchiveEntry(name, revision)

	resp, err := c.etcdClient.Get(context.Background(), keyspace, &client.GetOptions{Sort: false, Recursive: false, Quorum: true})
	if err != nil {
		return "", errors.EtcdToErrored(err)
	}

	return resp.Node.Value, nil
}
Example #9
0
// GetVolumeRuntime retrieves only the runtime parameters for the volume.
func (c *Client) GetVolumeRuntime(policy, name string) (RuntimeOptions, error) {
	runtime := RuntimeOptions{}

	resp, err := c.etcdClient.Get(context.Background(), c.volume(policy, name, "runtime"), nil)
	if err != nil {
		return runtime, errors.EtcdToErrored(err)
	}

	return runtime, json.Unmarshal([]byte(resp.Node.Value), &runtime)
}
Example #10
0
// Get retrieves the item from etcd's key/value store and then populates obj with its data.
func (c *Client) Get(obj db.Entity) error {
	return helpers.WrapGet(c, obj, func(path string) (string, []byte, error) {
		resp, err := c.client.Get(context.Background(), c.qualified(path), nil)
		if err != nil {
			return "", nil, errors.EtcdToErrored(err)
		}

		return resp.Node.Key, []byte(resp.Node.Value), nil
	})
}
Example #11
0
// CreatePolicyRevision creates an revision entry in a policy's history.
func (c *Client) CreatePolicyRevision(name string, policy string) error {
	timestamp := fmt.Sprint(time.Now().Unix())
	key := c.policyArchiveEntry(name, timestamp)

	_, err := c.etcdClient.Set(context.Background(), key, policy, nil)
	if err != nil {
		return errors.EtcdToErrored(err)
	}

	return nil
}
Example #12
0
// ListVolumes returns a map of volume name -> Volume.
func (c *Client) ListVolumes(policy string) (map[string]*Volume, error) {
	policyPath := c.prefixed(rootVolume, policy)

	resp, err := c.etcdClient.Get(context.Background(), policyPath, &client.GetOptions{Recursive: true, Sort: true})
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	configs := map[string]*Volume{}

	for _, node := range resp.Node.Nodes {
		if len(node.Nodes) > 0 {
			node = node.Nodes[0]
			key := strings.TrimPrefix(node.Key, policyPath)
			if !node.Dir && strings.HasSuffix(node.Key, "/create") {
				key = strings.TrimSuffix(key, "/create")

				config, ok := configs[key[1:]]
				if !ok {
					config = new(Volume)
				}

				if err := json.Unmarshal([]byte(node.Value), config); err != nil {
					return nil, err
				}
				// trim leading slash
				configs[key[1:]] = config
			}

			if !node.Dir && strings.HasSuffix(node.Key, "/runtime") {
				key = strings.TrimSuffix(key, "/create")

				config, ok := configs[key[1:]]
				if !ok {
					config = new(Volume)
				}

				if err := json.Unmarshal([]byte(node.Value), &config.RuntimeOptions); err != nil {
					return nil, err
				}
				// trim leading slash
				configs[key[1:]] = config
			}
		}
	}

	for _, config := range configs {
		if _, err := config.CreateOptions.ActualSize(); err != nil {
			return nil, err
		}
	}

	return configs, nil
}
Example #13
0
// ListAllVolumes returns an array with all the named policies and volumes the
// apiserver knows about. Volumes have syntax: policy/volumeName which will be
// reflected in the returned string.
func (c *Client) ListAllVolumes() ([]string, error) {
	resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootVolume), &client.GetOptions{Recursive: true, Sort: true})
	if err != nil {
		if er, ok := errors.EtcdToErrored(err).(*errored.Error); ok && er.Contains(errors.NotExists) {
			return []string{}, nil
		}

		return nil, errors.EtcdToErrored(err)
	}

	ret := []string{}

	for _, node := range resp.Node.Nodes {
		for _, innerNode := range node.Nodes {
			ret = append(ret, path.Join(path.Base(node.Key), path.Base(innerNode.Key)))
		}
	}

	return ret, nil
}
Example #14
0
// GetUse retrieves the UseMount for the given volume name.
func (c *Client) GetUse(ut UseLocker, vc *Volume) error {
	resp, err := c.etcdClient.Get(context.Background(), c.use(ut.Type(), vc.String()), nil)
	if err != nil {
		return errors.EtcdToErrored(err)
	}

	if err := json.Unmarshal([]byte(resp.Node.Value), ut); err != nil {
		return err
	}

	return nil
}
Example #15
0
// PublishGlobal publishes the global configuration.
func (tlc *Client) PublishGlobal(g *Global) error {
	gcPath := tlc.prefixed("global-config")

	value, err := json.Marshal(g.Canonical())
	if err != nil {
		return err
	}

	if _, err := tlc.etcdClient.Set(context.Background(), gcPath, string(value), &client.SetOptions{PrevExist: client.PrevIgnore}); err != nil {
		return errors.EtcdToErrored(err)
	}

	return nil
}
Example #16
0
// Delete removes the object from the store.
func (c *Client) Delete(obj db.Entity) error {
	if obj.Hooks().PreDelete != nil {
		if err := obj.Hooks().PreDelete(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	path, err := obj.Path()
	if err != nil {
		return err
	}

	if _, err := c.client.Delete(context.Background(), c.qualified(path), nil); err != nil {
		return errors.EtcdToErrored(err)
	}

	if obj.Hooks().PostDelete != nil {
		if err := obj.Hooks().PostDelete(c, obj); err != nil {
			return errors.EtcdToErrored(err)
		}
	}

	return nil
}
Example #17
0
// RemoveUse will remove a user from etcd. Does not fail if the user does
// not exist.
func (c *Client) RemoveUse(ut UseLocker, force bool) error {
	content, err := json.Marshal(ut)
	if err != nil {
		return err
	}

	logrus.Debugf("Removing Use Lock: %#v", ut)

	opts := &client.DeleteOptions{PrevValue: string(content)}
	if force {
		opts = nil
	}

	_, err = c.etcdClient.Delete(context.Background(), c.use(ut.Type(), ut.GetVolume()), opts)
	return errors.EtcdToErrored(err)
}
Example #18
0
// PublishVolumeRuntime publishes the runtime parameters for each volume.
func (c *Client) PublishVolumeRuntime(vo *Volume, ro RuntimeOptions) error {
	if err := ro.ValidateJSON(); err != nil {
		return errors.ErrJSONValidation.Combine(err)
	}

	content, err := json.Marshal(ro)
	if err != nil {
		return err
	}

	c.etcdClient.Set(context.Background(), c.prefixed(rootVolume, vo.PolicyName, vo.VolumeName), "", &client.SetOptions{Dir: true})
	if _, err := c.etcdClient.Set(context.Background(), c.volume(vo.PolicyName, vo.VolumeName, "runtime"), string(content), nil); err != nil {
		return errors.EtcdToErrored(err)
	}

	return nil
}
Example #19
0
// CurrentSchemaVersion returns the version of the last migration which was successfully run.
// If no previous migrations have been run, the schema version is 0.
// If there's any error besides "key doesn't exist", execution is aborted.
func (e *Engine) CurrentSchemaVersion() int64 {
	resp, err := e.etcdClient.Get(context.Background(), path.Join(e.prefix, backend.SchemaVersionKey), nil)
	if err != nil {
		if err := errors.EtcdToErrored(err); err != nil && err == errors.NotExists {
			return 0 // no key = schema version 0
		}

		logrus.Fatalf("Unexpected error when looking up schema version: %v\n", err)
	}

	i, err := strconv.Atoi(resp.Node.Value)
	if err != nil {
		logrus.Fatalf("Got back unexpected schema version data: %v\n", resp.Node.Value)
	}

	return int64(i)
}
Example #20
0
// ListPolicies provides an array of strings corresponding to the name of each
// policy.
func (c *Client) ListPolicies() ([]Policy, error) {
	resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootPolicy), &client.GetOptions{Recursive: true, Sort: true})
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	policies := []Policy{}
	for _, node := range resp.Node.Nodes {
		policy := Policy{}
		if err := json.Unmarshal([]byte(node.Value), &policy); err != nil {
			return nil, err
		}
		policies = append(policies, policy)
	}

	return policies, nil
}
Example #21
0
// ListPolicyRevisions returns a list of all the revisions for a given policy.
func (c *Client) ListPolicyRevisions(name string) ([]string, error) {
	keyspace := c.policyArchive(name)

	resp, err := c.etcdClient.Get(context.Background(), keyspace, &client.GetOptions{Sort: true, Recursive: false, Quorum: true})
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	ret := []string{}

	for _, node := range resp.Node.Nodes {
		_, revision := path.Split(node.Key)

		ret = append(ret, revision)
	}

	return ret, nil
}
Example #22
0
// ListUses lists the items in use.
func (c *Client) ListUses(typ string) ([]string, error) {
	resp, err := c.etcdClient.Get(context.Background(), c.prefixed(rootUse, typ), &client.GetOptions{Sort: true, Recursive: true})
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	ret := []string{}

	for _, node := range resp.Node.Nodes {
		for _, inner := range node.Nodes {
			key := path.Join(strings.TrimPrefix(inner.Key, c.prefixed(rootUse, typ)))
			// trim leading slash
			key = key[1:]
			ret = append(ret, key)
		}
	}

	return ret, nil
}
Example #23
0
// GetPolicy retrieves a policy from the configuration store.
func (c *Client) GetPolicy(name string) (*Policy, error) {
	if name == "" {
		return nil, errored.Errorf("Policy invalid: empty string for name")
	}

	resp, err := c.etcdClient.Get(context.Background(), c.policy(name), nil)
	if err != nil {
		return nil, errors.EtcdToErrored(err)
	}

	tc := NewPolicy()
	if err := json.Unmarshal([]byte(resp.Node.Value), tc); err != nil {
		return nil, err
	}

	tc.Name = name

	err = tc.Validate()
	return tc, err
}
Example #24
0
func (dc *DaemonConfig) loop() {
	for {
		time.Sleep(time.Second)

		// XXX this copy is so we can free the mutex quickly for more additions
		volumeCopy := map[string]*config.Volume{}

		volumeMutex.Lock()
		for volume, val := range volumes {
			logrus.Debugf("Adding volume %q for processing", volume)
			val2 := *val
			volumeCopy[volume] = &val2
		}
		volumeMutex.Unlock()

		for volume, val := range volumeCopy {
			if val.RuntimeOptions.UseSnapshots {
				freq, err := time.ParseDuration(val.RuntimeOptions.Snapshot.Frequency)
				if err != nil {
					logrus.Errorf("Volume %q has an invalid frequency. Skipping snapshot.", volume)
				}

				if time.Now().Unix()%int64(freq.Seconds()) == 0 {
					var isUsed bool
					var err error
					if isUsed, err = dc.Config.IsVolumeInUse(val, dc.Global); err != nil {
						logrus.Errorf("etcd error: %s", errors.EtcdToErrored(err)) // some issue with "etcd GET"; we should not hit this case
					}

					go func(val *config.Volume, isUsed bool) {
						// XXX we still want to prune snapshots even if the volume is not in use.
						if isUsed {
							dc.createSnapshot(val)
						}
						dc.pruneSnapshots(val)
					}(val, isUsed)
				}
			}
		}
	}
}
Example #25
0
// NewClient creates a new Client with connections to etcd already established.
func NewClient(hosts []string, prefix string) (*Client, error) {
	ec, err := client.New(client.Config{Endpoints: hosts})
	if err != nil {
		return nil, err
	}

	c := &Client{
		client:   client.NewKeysAPI(ec),
		prefix:   prefix,
		watchers: map[string]chan struct{}{},
	}

	if _, err := c.client.Set(context.Background(), c.prefix, "", &client.SetOptions{Dir: true, PrevExist: client.PrevNoExist}); err != nil {
		if err != nil {
			er, ok := errors.EtcdToErrored(err).(*errored.Error)
			if !ok || !er.Contains(errors.Exists) {
				return nil, errored.New("Initial setup").Combine(err)
			}
		}
	}

	return c, nil
}
Example #26
0
// RemoveTakeSnapshot removes a reference to a taken snapshot, intended to be used by volsupervisor
func (c *Client) RemoveTakeSnapshot(name string) error {
	_, err := c.etcdClient.Delete(context.Background(), c.prefixed(rootSnapshots, name), nil)
	return errors.EtcdToErrored(err)
}
Example #27
0
// RemoveVolume removes a volume from configuration.
func (c *Client) RemoveVolume(policy, name string) error {
	logrus.Debugf("Removing volume %s/%s from database", policy, name)
	_, err := c.etcdClient.Delete(context.Background(), c.prefixed(rootVolume, policy, name), &client.DeleteOptions{Recursive: true})
	return errors.EtcdToErrored(err)
}
Example #28
0
// DumpTarball dumps all the keys under the current etcd prefix into a
// gzip'd tarball'd directory-based representation of the namespace.
func (c *Client) DumpTarball() (string, error) {
	resp, err := c.etcdClient.Get(context.Background(), c.prefix, &client.GetOptions{Sort: true, Recursive: true, Quorum: true})
	if err != nil {
		return "", errored.Errorf(`Failed to recursively GET "%s" namespace from etcd`, c.prefix).Combine(errors.EtcdToErrored(err))
	}

	now := time.Now()

	// tar hangs during unpacking if the base directory has colons in it
	// unless --force-local is specified, so use the simpler "%Y%m%d-%H%M%S".
	niceTimeFormat := fmt.Sprintf("%d%02d%02d-%02d%02d%02d",
		now.Year(), now.Month(), now.Day(),
		now.Hour(), now.Minute(), now.Second())

	file, err := ioutil.TempFile("", "etcd_dump_"+niceTimeFormat+"_")
	if err != nil {
		return "", errored.Errorf("Failed to create tempfile").Combine(err)
	}
	defer file.Close()

	// create a gzipped, tarball writer which cleans up after itself
	gzipWriter := gzip.NewWriter(file)
	defer gzipWriter.Close()

	tarWriter := tar.NewWriter(gzipWriter)
	defer tarWriter.Close()

	// ensure that the tarball extracts to a folder with the same name as the tarball
	baseDirectory := filepath.Base(file.Name())

	err = addNodeToTarball(resp.Node, tarWriter, baseDirectory)
	if err != nil {
		return "", err
	}

	// give the file a more fitting name
	newFilename := file.Name() + ".tar.gz"

	err = os.Rename(file.Name(), newFilename)
	if err != nil {
		return "", err
	}

	return newFilename, nil
}
Example #29
0
// DeletePolicy removes a policy from the configuration store.
func (c *Client) DeletePolicy(name string) error {
	_, err := c.etcdClient.Delete(context.Background(), c.policy(name), nil)
	return errors.EtcdToErrored(err)
}