// compact compacts etcd store and returns current rev. // It will return the current compact time and global revision if no error occurred. // Note that CAS fail will not incur any error. func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, int64, error) { resp, err := client.KV.Txn(ctx).If( clientv3.Compare(clientv3.Version(compactRevKey), "=", t), ).Then( clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10)), // Expect side effect: increment Version ).Else( clientv3.OpGet(compactRevKey), ).Commit() if err != nil { return t, rev, err } curRev := resp.Header.Revision if !resp.Succeeded { curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version return curTime, curRev, nil } curTime := t + 1 if rev == 0 { // We don't compact on bootstrap. return curTime, curRev, nil } if _, err = client.Compact(ctx, rev); err != nil { return curTime, curRev, err } glog.Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints()) return curTime, curRev, nil }
func (b *blockEtcd) CreateBlockVolume(volume *models.Volume) error { new, err := b.AtomicModifyKey([]byte(etcd.MkKey("meta", "volumeminter")), etcd.BytesAddOne) volume.Id = new.(uint64) if err != nil { return err } vbytes, err := volume.Marshal() if err != nil { return err } inodeBytes := torus.NewINodeRef(torus.VolumeID(volume.Id), 1).ToBytes() do := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(etcd.MkKey("volumes", volume.Name)), "=", 0), ).Then( etcdv3.OpPut(etcd.MkKey("volumes", volume.Name), string(etcd.Uint64ToBytes(volume.Id))), etcdv3.OpPut(etcd.MkKey("volumeid", etcd.Uint64ToHex(volume.Id)), string(vbytes)), etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "inode"), string(etcd.Uint64ToBytes(1))), etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "blockinode"), string(inodeBytes)), ) resp, err := do.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrExists } return nil }
func (c *etcdCtx) SetRing(ring torus.Ring) error { oldr, etcdver, err := c.getRing() if err != nil { return err } if oldr.Version() != ring.Version()-1 { return torus.ErrNonSequentialRing } b, err := ring.Marshal() if err != nil { return err } key := MkKey("meta", "the-one-ring") txn := c.etcd.Client.Txn(c.getContext()).If( etcdv3.Compare(etcdv3.Version(key), "=", etcdver), ).Then( etcdv3.OpPut(key, string(b)), ) resp, err := txn.Commit() if err != nil { return err } if resp.Succeeded { return nil } return torus.ErrAgain }
func (b *blockEtcd) SaveSnapshot(name string) error { vid := uint64(b.vid) for { sshotKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name) inoKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode") tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(sshotKey), "=", 0), ).Then( etcdv3.OpGet(inoKey), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrExists } v := resp.Responses[0].GetResponseRange().Kvs[0] inode := Snapshot{ Name: name, When: time.Now(), INodeRef: v.Value, } bytes, err := json.Marshal(inode) if err != nil { return err } tx = b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(inoKey), "=", v.Version), ).Then( etcdv3.OpPut(sshotKey, string(bytes)), ) resp, err = tx.Commit() if err != nil { return err } if !resp.Succeeded { continue } return nil } }
// putNewKV attempts to create the given key, only succeeding if the key did // not yet exist. func putNewKV(kv v3.KV, key, val string, leaseID v3.LeaseID) (int64, error) { cmp := v3.Compare(v3.Version(key), "=", 0) req := v3.OpPut(key, val, v3.WithLease(leaseID)) txnresp, err := kv.Txn(context.TODO()).If(cmp).Then(req).Commit() if err != nil { return 0, err } if !txnresp.Succeeded { return 0, ErrKeyExists } return txnresp.Header.Revision, nil }
func parseCompare(line string) (*clientv3.Cmp, error) { var ( key string op string val string ) lparenSplit := strings.SplitN(line, "(", 2) if len(lparenSplit) != 2 { return nil, fmt.Errorf("malformed comparison: %s", line) } target := lparenSplit[0] n, serr := fmt.Sscanf(lparenSplit[1], "%q) %s %q", &key, &op, &val) if n != 3 { return nil, fmt.Errorf("malformed comparison: %s; got %s(%q) %s %q", line, target, key, op, val) } if serr != nil { return nil, fmt.Errorf("malformed comparison: %s (%v)", line, serr) } var ( v int64 err error cmp clientv3.Cmp ) switch target { case "ver", "version": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.Version(key), op, v) } case "c", "create": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.CreateRevision(key), op, v) } case "m", "mod": if v, err = strconv.ParseInt(val, 10, 64); err == nil { cmp = clientv3.Compare(clientv3.ModRevision(key), op, v) } case "val", "value": cmp = clientv3.Compare(clientv3.Value(key), op, val) default: return nil, fmt.Errorf("malformed comparison: %s (unknown target %s)", line, target) } if err != nil { return nil, fmt.Errorf("invalid txn compare request: %s", line) } return &cmp, nil }
func (b *blockEtcd) DeleteSnapshot(name string) error { vid := uint64(b.vid) k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name) tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(k), ">", 0), ).Then( etcdv3.OpDelete(k), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrLocked } return nil }
func (b *blockEtcd) Unlock() error { vid := uint64(b.vid) k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock") tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(k), ">", 0), etcdv3.Compare(etcdv3.Value(k), "=", b.Etcd.UUID()), ).Then( etcdv3.OpDelete(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrLocked } return nil }
func (b *blockEtcd) Lock(lease int64) error { if lease == 0 { return torus.ErrInvalid } k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(uint64(b.vid)), "blocklock") tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(k), "=", 0), ).Then( etcdv3.OpPut(k, b.Etcd.UUID(), etcdv3.WithLease(etcdv3.LeaseID(lease))), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrLocked } return nil }
func (b *blockEtcd) DeleteVolume() error { vid := uint64(b.vid) tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")), "=", 0), ).Then( etcdv3.OpDelete(etcd.MkKey("volumes", b.name)), etcdv3.OpDelete(etcd.MkKey("volumeid", etcd.Uint64ToHex(vid))), etcdv3.OpDelete(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid)), etcdv3.WithPrefix()), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrLocked } return nil }
func (b *blockEtcd) SyncINode(inode torus.INodeRef) error { vid := uint64(inode.Volume()) inodeBytes := string(inode.ToBytes()) k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock") tx := b.Etcd.Client.Txn(b.getContext()).If( etcdv3.Compare(etcdv3.Version(k), ">", 0), etcdv3.Compare(etcdv3.Value(k), "=", b.Etcd.UUID()), ).Then( etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode"), inodeBytes), ) resp, err := tx.Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrLocked } return nil }
func initEtcdMetadata(cfg torus.Config, gmd torus.GlobalMetadata, ringType torus.RingType) error { gmdbytes, err := json.Marshal(gmd) if err != nil { return err } emptyRing, err := ring.CreateRing(&models.Ring{ Type: uint32(ringType), Version: 1, ReplicationFactor: 2, }) if err != nil { return err } ringb, err := emptyRing.Marshal() if err != nil { return err } client, err := etcdv3.New(etcdv3.Config{Endpoints: []string{cfg.MetadataAddress}, TLS: cfg.TLS}) if err != nil { return err } defer client.Close() txn := client.Txn(context.Background()) resp, err := txn.If( etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), "=", 0), ).Then( etcdv3.OpPut(MkKey("meta", "volumeminter"), string(Uint64ToBytes(1))), etcdv3.OpPut(MkKey("meta", "globalmetadata"), string(gmdbytes)), ).Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrExists } _, err = client.Put(context.Background(), MkKey("meta", "the-one-ring"), string(ringb)) if err != nil { return err } return nil }
func (c *etcdCtx) AtomicModifyKey(k []byte, f AtomicModifyFunc) (interface{}, error) { key := string(k) resp, err := c.etcd.Client.Get(c.getContext(), key) if err != nil { return nil, err } var version int64 var value []byte if len(resp.Kvs) != 1 { version = 0 value = []byte{} } else { kv := resp.Kvs[0] version = kv.Version value = kv.Value } for { newBytes, fval, err := f(value) if err != nil { return nil, err } txn := c.etcd.Client.Txn(c.getContext()).If( etcdv3.Compare(etcdv3.Version(key), "=", version), ).Then( etcdv3.OpPut(key, string(newBytes)), ).Else( etcdv3.OpGet(key), ) resp, err := txn.Commit() if err != nil { return nil, err } if resp.Succeeded { return fval, nil } promAtomicRetries.WithLabelValues(string(key)).Inc() kv := resp.Responses[0].GetResponseRange().Kvs[0] version = kv.Version value = kv.Value } }
func (e *Etcd) getGlobalMetadata() error { txn := e.Client.Txn(context.Background()) resp, err := txn.If( etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), ">", 0), ).Then( etcdv3.OpGet(MkKey("meta", "globalmetadata")), ).Commit() if err != nil { return err } if !resp.Succeeded { return torus.ErrNoGlobalMetadata } var gmd torus.GlobalMetadata err = json.Unmarshal(resp.Responses[0].GetResponseRange().Kvs[0].Value, &gmd) if err != nil { return err } e.global = gmd return nil }