// compact compacts etcd store and returns current rev.
// It will return the current compact time and global revision if no error occurred.
// Note that CAS fail will not incur any error.
func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, int64, error) {
resp, err := client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.Version(compactRevKey), "=", t),
).Then(
clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10)), // Expect side effect: increment Version
).Else(
clientv3.OpGet(compactRevKey),
).Commit()
if err != nil {
return t, rev, err
}
curRev := resp.Header.Revision
if !resp.Succeeded {
curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version
return curTime, curRev, nil
}
curTime := t + 1
if rev == 0 {
// We don't compact on bootstrap.
return curTime, curRev, nil
}
if _, err = client.Compact(ctx, rev); err != nil {
return curTime, curRev, err
}
glog.Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints())
return curTime, curRev, nil
}
func (b *blockEtcd) CreateBlockVolume(volume *models.Volume) error {
new, err := b.AtomicModifyKey([]byte(etcd.MkKey("meta", "volumeminter")), etcd.BytesAddOne)
volume.Id = new.(uint64)
if err != nil {
return err
}
vbytes, err := volume.Marshal()
if err != nil {
return err
}
inodeBytes := torus.NewINodeRef(torus.VolumeID(volume.Id), 1).ToBytes()
do := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(etcd.MkKey("volumes", volume.Name)), "=", 0),
).Then(
etcdv3.OpPut(etcd.MkKey("volumes", volume.Name), string(etcd.Uint64ToBytes(volume.Id))),
etcdv3.OpPut(etcd.MkKey("volumeid", etcd.Uint64ToHex(volume.Id)), string(vbytes)),
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "inode"), string(etcd.Uint64ToBytes(1))),
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "blockinode"), string(inodeBytes)),
)
resp, err := do.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
return nil
}
func (c *etcdCtx) SetRing(ring torus.Ring) error {
oldr, etcdver, err := c.getRing()
if err != nil {
return err
}
if oldr.Version() != ring.Version()-1 {
return torus.ErrNonSequentialRing
}
b, err := ring.Marshal()
if err != nil {
return err
}
key := MkKey("meta", "the-one-ring")
txn := c.etcd.Client.Txn(c.getContext()).If(
etcdv3.Compare(etcdv3.Version(key), "=", etcdver),
).Then(
etcdv3.OpPut(key, string(b)),
)
resp, err := txn.Commit()
if err != nil {
return err
}
if resp.Succeeded {
return nil
}
return torus.ErrAgain
}
func (b *blockEtcd) SaveSnapshot(name string) error {
vid := uint64(b.vid)
for {
sshotKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name)
inoKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(sshotKey), "=", 0),
).Then(
etcdv3.OpGet(inoKey),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
v := resp.Responses[0].GetResponseRange().Kvs[0]
inode := Snapshot{
Name: name,
When: time.Now(),
INodeRef: v.Value,
}
bytes, err := json.Marshal(inode)
if err != nil {
return err
}
tx = b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(inoKey), "=", v.Version),
).Then(
etcdv3.OpPut(sshotKey, string(bytes)),
)
resp, err = tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
continue
}
return nil
}
}
// putNewKV attempts to create the given key, only succeeding if the key did
// not yet exist.
func putNewKV(kv v3.KV, key, val string, leaseID v3.LeaseID) (int64, error) {
cmp := v3.Compare(v3.Version(key), "=", 0)
req := v3.OpPut(key, val, v3.WithLease(leaseID))
txnresp, err := kv.Txn(context.TODO()).If(cmp).Then(req).Commit()
if err != nil {
return 0, err
}
if !txnresp.Succeeded {
return 0, ErrKeyExists
}
return txnresp.Header.Revision, nil
}
func parseCompare(line string) (*clientv3.Cmp, error) {
var (
key string
op string
val string
)
lparenSplit := strings.SplitN(line, "(", 2)
if len(lparenSplit) != 2 {
return nil, fmt.Errorf("malformed comparison: %s", line)
}
target := lparenSplit[0]
n, serr := fmt.Sscanf(lparenSplit[1], "%q) %s %q", &key, &op, &val)
if n != 3 {
return nil, fmt.Errorf("malformed comparison: %s; got %s(%q) %s %q", line, target, key, op, val)
}
if serr != nil {
return nil, fmt.Errorf("malformed comparison: %s (%v)", line, serr)
}
var (
v int64
err error
cmp clientv3.Cmp
)
switch target {
case "ver", "version":
if v, err = strconv.ParseInt(val, 10, 64); err == nil {
cmp = clientv3.Compare(clientv3.Version(key), op, v)
}
case "c", "create":
if v, err = strconv.ParseInt(val, 10, 64); err == nil {
cmp = clientv3.Compare(clientv3.CreateRevision(key), op, v)
}
case "m", "mod":
if v, err = strconv.ParseInt(val, 10, 64); err == nil {
cmp = clientv3.Compare(clientv3.ModRevision(key), op, v)
}
case "val", "value":
cmp = clientv3.Compare(clientv3.Value(key), op, val)
default:
return nil, fmt.Errorf("malformed comparison: %s (unknown target %s)", line, target)
}
if err != nil {
return nil, fmt.Errorf("invalid txn compare request: %s", line)
}
return &cmp, nil
}
func (b *blockEtcd) DeleteSnapshot(name string) error {
vid := uint64(b.vid)
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name)
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), ">", 0),
).Then(
etcdv3.OpDelete(k),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func (b *blockEtcd) Unlock() error {
vid := uint64(b.vid)
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), ">", 0),
etcdv3.Compare(etcdv3.Value(k), "=", b.Etcd.UUID()),
).Then(
etcdv3.OpDelete(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func (b *blockEtcd) Lock(lease int64) error {
if lease == 0 {
return torus.ErrInvalid
}
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(uint64(b.vid)), "blocklock")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), "=", 0),
).Then(
etcdv3.OpPut(k, b.Etcd.UUID(), etcdv3.WithLease(etcdv3.LeaseID(lease))),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func (b *blockEtcd) DeleteVolume() error {
vid := uint64(b.vid)
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")), "=", 0),
).Then(
etcdv3.OpDelete(etcd.MkKey("volumes", b.name)),
etcdv3.OpDelete(etcd.MkKey("volumeid", etcd.Uint64ToHex(vid))),
etcdv3.OpDelete(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid)), etcdv3.WithPrefix()),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func (b *blockEtcd) SyncINode(inode torus.INodeRef) error {
vid := uint64(inode.Volume())
inodeBytes := string(inode.ToBytes())
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), ">", 0),
etcdv3.Compare(etcdv3.Value(k), "=", b.Etcd.UUID()),
).Then(
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode"), inodeBytes),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func initEtcdMetadata(cfg torus.Config, gmd torus.GlobalMetadata, ringType torus.RingType) error {
gmdbytes, err := json.Marshal(gmd)
if err != nil {
return err
}
emptyRing, err := ring.CreateRing(&models.Ring{
Type: uint32(ringType),
Version: 1,
ReplicationFactor: 2,
})
if err != nil {
return err
}
ringb, err := emptyRing.Marshal()
if err != nil {
return err
}
client, err := etcdv3.New(etcdv3.Config{Endpoints: []string{cfg.MetadataAddress}, TLS: cfg.TLS})
if err != nil {
return err
}
defer client.Close()
txn := client.Txn(context.Background())
resp, err := txn.If(
etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), "=", 0),
).Then(
etcdv3.OpPut(MkKey("meta", "volumeminter"), string(Uint64ToBytes(1))),
etcdv3.OpPut(MkKey("meta", "globalmetadata"), string(gmdbytes)),
).Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
_, err = client.Put(context.Background(), MkKey("meta", "the-one-ring"), string(ringb))
if err != nil {
return err
}
return nil
}
func (c *etcdCtx) AtomicModifyKey(k []byte, f AtomicModifyFunc) (interface{}, error) {
key := string(k)
resp, err := c.etcd.Client.Get(c.getContext(), key)
if err != nil {
return nil, err
}
var version int64
var value []byte
if len(resp.Kvs) != 1 {
version = 0
value = []byte{}
} else {
kv := resp.Kvs[0]
version = kv.Version
value = kv.Value
}
for {
newBytes, fval, err := f(value)
if err != nil {
return nil, err
}
txn := c.etcd.Client.Txn(c.getContext()).If(
etcdv3.Compare(etcdv3.Version(key), "=", version),
).Then(
etcdv3.OpPut(key, string(newBytes)),
).Else(
etcdv3.OpGet(key),
)
resp, err := txn.Commit()
if err != nil {
return nil, err
}
if resp.Succeeded {
return fval, nil
}
promAtomicRetries.WithLabelValues(string(key)).Inc()
kv := resp.Responses[0].GetResponseRange().Kvs[0]
version = kv.Version
value = kv.Value
}
}
func (e *Etcd) getGlobalMetadata() error {
txn := e.Client.Txn(context.Background())
resp, err := txn.If(
etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), ">", 0),
).Then(
etcdv3.OpGet(MkKey("meta", "globalmetadata")),
).Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrNoGlobalMetadata
}
var gmd torus.GlobalMetadata
err = json.Unmarshal(resp.Responses[0].GetResponseRange().Kvs[0].Value, &gmd)
if err != nil {
return err
}
e.global = gmd
return nil
}