// compact compacts etcd store and returns current rev.
// It will return the current compact time and global revision if no error occurred.
// Note that CAS fail will not incur any error.
func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, int64, error) {
resp, err := client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.Version(compactRevKey), "=", t),
).Then(
clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10)), // Expect side effect: increment Version
).Else(
clientv3.OpGet(compactRevKey),
).Commit()
if err != nil {
return t, rev, err
}
curRev := resp.Header.Revision
if !resp.Succeeded {
curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version
return curTime, curRev, nil
}
curTime := t + 1
if rev == 0 {
// We don't compact on bootstrap.
return curTime, curRev, nil
}
if _, err = client.Compact(ctx, rev); err != nil {
return curTime, curRev, err
}
glog.Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints())
return curTime, curRev, nil
}
func (b *blockEtcd) CreateBlockVolume(volume *models.Volume) error {
new, err := b.AtomicModifyKey([]byte(etcd.MkKey("meta", "volumeminter")), etcd.BytesAddOne)
volume.Id = new.(uint64)
if err != nil {
return err
}
vbytes, err := volume.Marshal()
if err != nil {
return err
}
inodeBytes := torus.NewINodeRef(torus.VolumeID(volume.Id), 1).ToBytes()
do := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(etcd.MkKey("volumes", volume.Name)), "=", 0),
).Then(
etcdv3.OpPut(etcd.MkKey("volumes", volume.Name), string(etcd.Uint64ToBytes(volume.Id))),
etcdv3.OpPut(etcd.MkKey("volumeid", etcd.Uint64ToHex(volume.Id)), string(vbytes)),
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "inode"), string(etcd.Uint64ToBytes(1))),
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(volume.Id), "blockinode"), string(inodeBytes)),
)
resp, err := do.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
return nil
}
// Lock locks the mutex with a cancellable context. If the context is cancelled
// while trying to acquire the lock, the mutex tries to clean its stale lock entry.
func (m *Mutex) Lock(ctx context.Context) error {
s, serr := NewSession(m.client)
if serr != nil {
return serr
}
m.myKey = fmt.Sprintf("%s/%x", m.pfx, s.Lease())
cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0)
// put self in lock waiters via myKey; oldest waiter holds lock
put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease()))
// reuse key in case this session already holds the lock
get := v3.OpGet(m.myKey)
resp, err := m.client.Txn(ctx).If(cmp).Then(put).Else(get).Commit()
if err != nil {
return err
}
m.myRev = resp.Header.Revision
if !resp.Succeeded {
m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
}
// wait for deletion revisions prior to myKey
err = waitDeletes(ctx, m.client, m.pfx, v3.WithPrefix(), v3.WithRev(m.myRev-1))
// release lock key if cancelled
select {
case <-ctx.Done():
m.Unlock()
default:
}
return err
}
func TestTxnWriteFail(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.Client(0))
clus.Members[0].Stop(t)
<-clus.Members[0].StopNotify()
resp, err := kv.Txn().Then(clientv3.OpPut("foo", "bar", 0)).Commit()
if err == nil {
t.Fatalf("expected error, got response %v", resp)
}
// reconnect so cluster terminate doesn't complain about double-close
clus.Members[0].Restart(t)
// and ensure the put didn't take
gresp, gerr := kv.Get("foo", 0)
if gerr != nil {
t.Fatal(gerr)
}
if len(gresp.Kvs) != 0 {
t.Fatalf("expected no keys, got %v", gresp.Kvs)
}
}
func parseRequestUnion(line string) (*clientv3.Op, error) {
args := argify(line)
if len(args) < 2 {
return nil, fmt.Errorf("invalid txn compare request: %s", line)
}
opc := make(chan clientv3.Op, 1)
put := NewPutCommand()
put.Run = func(cmd *cobra.Command, args []string) {
key, value, opts := getPutOp(cmd, args)
opc <- clientv3.OpPut(key, value, opts...)
}
get := NewGetCommand()
get.Run = func(cmd *cobra.Command, args []string) {
key, opts := getGetOp(cmd, args)
opc <- clientv3.OpGet(key, opts...)
}
del := NewDelCommand()
del.Run = func(cmd *cobra.Command, args []string) {
key, opts := getDelOp(cmd, args)
opc <- clientv3.OpDelete(key, opts...)
}
cmds := &cobra.Command{SilenceErrors: true}
cmds.AddCommand(put, get, del)
cmds.SetArgs(args)
if err := cmds.Execute(); err != nil {
return nil, fmt.Errorf("invalid txn request: %s", line)
}
op := <-opc
return &op, nil
}
func (c *etcdCtx) SetRing(ring torus.Ring) error {
oldr, etcdver, err := c.getRing()
if err != nil {
return err
}
if oldr.Version() != ring.Version()-1 {
return torus.ErrNonSequentialRing
}
b, err := ring.Marshal()
if err != nil {
return err
}
key := MkKey("meta", "the-one-ring")
txn := c.etcd.Client.Txn(c.getContext()).If(
etcdv3.Compare(etcdv3.Version(key), "=", etcdver),
).Then(
etcdv3.OpPut(key, string(b)),
)
resp, err := txn.Commit()
if err != nil {
return err
}
if resp.Succeeded {
return nil
}
return torus.ErrAgain
}
// Create implements storage.Interface.Create.
func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64) error {
if version, err := s.versioner.ObjectResourceVersion(obj); err == nil && version != 0 {
return errors.New("resourceVersion should not be set on objects to be created")
}
data, err := runtime.Encode(s.codec, obj)
if err != nil {
return err
}
key = keyWithPrefix(s.pathPrefix, key)
opts, err := s.ttlOpts(ctx, int64(ttl))
if err != nil {
return err
}
txnResp, err := s.client.KV.Txn(ctx).If(
notFound(key),
).Then(
clientv3.OpPut(key, string(data), opts...),
).Commit()
if err != nil {
return err
}
if !txnResp.Succeeded {
return storage.NewKeyExistsError(key, 0)
}
if out != nil {
putResp := txnResp.Responses[0].GetResponsePut()
return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
}
return nil
}
func TestTxnWriteFail(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.Client(0))
ctx := context.TODO()
clus.Members[0].Stop(t)
<-clus.Members[0].StopNotify()
donec := make(chan struct{})
go func() {
resp, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar")).Commit()
if err == nil {
t.Fatalf("expected error, got response %v", resp)
}
donec <- struct{}{}
}()
dialTimeout := 5 * time.Second
select {
case <-time.After(2*dialTimeout + time.Second):
t.Fatalf("timed out waiting for txn to fail")
case <-donec:
// don't restart cluster until txn errors out
}
go func() {
// reconnect so terminate doesn't complain about double-close
clus.Members[0].Restart(t)
// wait for etcdserver to get established (CI races and get req times out)
time.Sleep(2 * time.Second)
donec <- struct{}{}
// and ensure the put didn't take
gresp, gerr := kv.Get(ctx, "foo")
if gerr != nil {
t.Fatal(gerr)
}
if len(gresp.Kvs) != 0 {
t.Fatalf("expected no keys, got %v", gresp.Kvs)
}
donec <- struct{}{}
}()
select {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for restart")
case <-donec:
}
select {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for get")
case <-donec:
}
}
// TestSTMSerialize tests that serialization is honored when serializable.
func TestSTMSerialize(t *testing.T) {
clus := NewClusterV3(t, &ClusterConfig{Size: 3})
defer clus.Terminate(t)
etcdc := clus.RandClient()
// set up initial keys
keys := make([]string, 5)
for i := 0; i < len(keys); i++ {
keys[i] = fmt.Sprintf("foo-%d", i)
}
// update keys in full batches
updatec := make(chan struct{})
go func() {
defer close(updatec)
for i := 0; i < 5; i++ {
s := fmt.Sprintf("%d", i)
ops := []v3.Op{}
for _, k := range keys {
ops = append(ops, v3.OpPut(k, s))
}
if _, err := etcdc.Txn(context.TODO()).Then(ops...).Commit(); err != nil {
t.Fatalf("couldn't put keys (%v)", err)
}
updatec <- struct{}{}
}
}()
// read all keys in txn, make sure all values match
errc := make(chan error)
for range updatec {
curEtcdc := clus.RandClient()
applyf := func(stm concurrency.STM) error {
vs := []string{}
for i := range keys {
vs = append(vs, stm.Get(keys[i]))
}
for i := range vs {
if vs[0] != vs[i] {
return fmt.Errorf("got vs[%d] = %v, want %v", i, vs[i], vs[0])
}
}
return nil
}
go func() {
_, err := concurrency.NewSTMSerializable(context.TODO(), curEtcdc, applyf)
errc <- err
}()
}
for i := 0; i < 5; i++ {
if err := <-errc; err != nil {
t.Error(err)
}
}
}
// GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
func (s *store) GuaranteedUpdate(ctx context.Context, key string, out runtime.Object, ignoreNotFound bool, precondtions *storage.Preconditions, tryUpdate storage.UpdateFunc) error {
v, err := conversion.EnforcePtr(out)
if err != nil {
panic("unable to convert output object to pointer")
}
key = keyWithPrefix(s.pathPrefix, key)
getResp, err := s.client.KV.Get(ctx, key)
if err != nil {
return err
}
for {
origState, err := s.getState(getResp, key, v, ignoreNotFound)
if err != nil {
return err
}
if err := checkPreconditions(key, precondtions, origState.obj); err != nil {
return err
}
ret, ttl, err := s.updateState(origState, tryUpdate)
if err != nil {
return err
}
data, err := runtime.Encode(s.codec, ret)
if err != nil {
return err
}
if bytes.Equal(data, origState.data) {
return decode(s.codec, s.versioner, origState.data, out, origState.rev)
}
opts, err := s.ttlOpts(ctx, int64(ttl))
if err != nil {
return err
}
txnResp, err := s.client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
).Then(
clientv3.OpPut(key, string(data), opts...),
).Else(
clientv3.OpGet(key),
).Commit()
if err != nil {
return err
}
if !txnResp.Succeeded {
getResp = (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
glog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key)
continue
}
putResp := txnResp.Responses[0].GetResponsePut()
return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
}
}
func initEtcdMetadata(cfg torus.Config, gmd torus.GlobalMetadata, ringType torus.RingType) error {
gmdbytes, err := json.Marshal(gmd)
if err != nil {
return err
}
emptyRing, err := ring.CreateRing(&models.Ring{
Type: uint32(ringType),
Version: 1,
ReplicationFactor: 2,
})
if err != nil {
return err
}
ringb, err := emptyRing.Marshal()
if err != nil {
return err
}
client, err := etcdv3.New(etcdv3.Config{Endpoints: []string{cfg.MetadataAddress}, TLS: cfg.TLS})
if err != nil {
return err
}
defer client.Close()
txn := client.Txn(context.Background())
resp, err := txn.If(
etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), "=", 0),
).Then(
etcdv3.OpPut(MkKey("meta", "volumeminter"), string(Uint64ToBytes(1))),
etcdv3.OpPut(MkKey("meta", "globalmetadata"), string(gmdbytes)),
).Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
_, err = client.Put(context.Background(), MkKey("meta", "the-one-ring"), string(ringb))
if err != nil {
return err
}
return nil
}
func putFunc(cmd *cobra.Command, args []string) {
if keySpaceSize <= 0 {
fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
os.Exit(1)
}
requests := make(chan v3.Op, totalClients)
clients := mustCreateClients(totalClients, totalConns)
k, v := make([]byte, keySize), string(mustRandBytes(valSize))
bar = pb.New(putTotal)
bar.Format("Bom !")
bar.Start()
r := newReport()
for i := range clients {
wg.Add(1)
go func(c *v3.Client) {
defer wg.Done()
for op := range requests {
st := time.Now()
_, err := c.Do(context.Background(), op)
r.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
bar.Increment()
}
}(clients[i])
}
go func() {
for i := 0; i < putTotal; i++ {
if seqKeys {
binary.PutVarint(k, int64(i%keySpaceSize))
} else {
binary.PutVarint(k, int64(rand.Intn(keySpaceSize)))
}
requests <- v3.OpPut(string(k), v)
}
close(requests)
}()
if compactInterval > 0 {
go func() {
for {
time.Sleep(compactInterval)
compactKV(clients)
}
}()
}
rc := r.Run()
wg.Wait()
close(r.Results())
bar.Finish()
fmt.Println(<-rc)
}
// putNewKV attempts to create the given key, only succeeding if the key did
// not yet exist.
func putNewKV(kv v3.KV, key, val string, leaseID v3.LeaseID) (int64, error) {
cmp := v3.Compare(v3.Version(key), "=", 0)
req := v3.OpPut(key, val, v3.WithLease(leaseID))
txnresp, err := kv.Txn(context.TODO()).If(cmp).Then(req).Commit()
if err != nil {
return 0, err
}
if !txnresp.Succeeded {
return 0, ErrKeyExists
}
return txnresp.Header.Revision, nil
}
func ExampleKV_do() {
cli, err := clientv3.New(clientv3.Config{
Endpoints: endpoints,
DialTimeout: dialTimeout,
})
if err != nil {
log.Fatal(err)
}
defer cli.Close()
ops := []clientv3.Op{
clientv3.OpPut("put-key", "123"),
clientv3.OpGet("put-key"),
clientv3.OpPut("put-key", "456")}
for _, op := range ops {
if _, err := cli.Do(context.TODO(), op); err != nil {
log.Fatal(err)
}
}
}
// newSequentialKV allocates a new sequential key <prefix>/nnnnn with a given
// value and lease. Note: a bookkeeping node __<prefix> is also allocated.
func newSequentialKV(kv v3.KV, prefix, val string, leaseID v3.LeaseID) (*RemoteKV, error) {
resp, err := kv.Get(context.TODO(), prefix, v3.WithLastKey()...)
if err != nil {
return nil, err
}
// add 1 to last key, if any
newSeqNum := 0
if len(resp.Kvs) != 0 {
fields := strings.Split(string(resp.Kvs[0].Key), "/")
_, serr := fmt.Sscanf(fields[len(fields)-1], "%d", &newSeqNum)
if serr != nil {
return nil, serr
}
newSeqNum++
}
newKey := fmt.Sprintf("%s/%016d", prefix, newSeqNum)
// base prefix key must be current (i.e., <=) with the server update;
// the base key is important to avoid the following:
// N1: LastKey() == 1, start txn.
// N2: New Key 2, New Key 3, Delete Key 2
// N1: txn succeeds allocating key 2 when it shouldn't
baseKey := "__" + prefix
// current revision might contain modification so +1
cmp := v3.Compare(v3.ModRevision(baseKey), "<", resp.Header.Revision+1)
reqPrefix := v3.OpPut(baseKey, "", v3.WithLease(leaseID))
reqNewKey := v3.OpPut(newKey, val, v3.WithLease(leaseID))
txn := kv.Txn(context.TODO())
txnresp, err := txn.If(cmp).Then(reqPrefix, reqNewKey).Commit()
if err != nil {
return nil, err
}
if !txnresp.Succeeded {
return newSequentialKV(kv, prefix, val, leaseID)
}
return &RemoteKV{kv, newKey, txnresp.Header.Revision, val}, nil
}
func putFunc(cmd *cobra.Command, args []string) {
if keySpaceSize <= 0 {
fmt.Fprintf(os.Stderr, "expected positive --key-space-size, got (%v)", keySpaceSize)
os.Exit(1)
}
results = make(chan result)
requests := make(chan v3.Op, totalClients)
bar = pb.New(putTotal)
k, v := make([]byte, keySize), string(mustRandBytes(valSize))
clients := mustCreateClients(totalClients, totalConns)
bar.Format("Bom !")
bar.Start()
for i := range clients {
wg.Add(1)
go doPut(context.Background(), clients[i], requests)
}
pdoneC := printReport(results)
go func() {
for i := 0; i < putTotal; i++ {
if seqKeys {
binary.PutVarint(k, int64(i%keySpaceSize))
} else {
binary.PutVarint(k, int64(rand.Intn(keySpaceSize)))
}
requests <- v3.OpPut(string(k), v)
}
close(requests)
}()
if compactInterval > 0 {
go func() {
for {
time.Sleep(compactInterval)
compactKV(clients)
}
}()
}
wg.Wait()
bar.Finish()
close(results)
<-pdoneC
}
func ExampleKV_txn() {
cli, err := clientv3.New(clientv3.Config{
Endpoints: endpoints,
DialTimeout: dialTimeout,
})
if err != nil {
log.Fatal(err)
}
defer cli.Close()
kvc := clientv3.NewKV(cli)
_, err = kvc.Put(context.TODO(), "key", "xyz")
if err != nil {
log.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
_, err = kvc.Txn(ctx).
If(clientv3.Compare(clientv3.Value("key"), ">", "abc")). // txn value comparisons are lexical
Then(clientv3.OpPut("key", "XYZ")). // this runs, since 'xyz' > 'abc'
Else(clientv3.OpPut("key", "ABC")).
Commit()
cancel()
if err != nil {
log.Fatal(err)
}
gresp, err := kvc.Get(context.TODO(), "key")
cancel()
if err != nil {
log.Fatal(err)
}
for _, ev := range gresp.Kvs {
fmt.Printf("%s : %s\n", ev.Key, ev.Value)
}
// key : XYZ
}
func TestTxnError(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.RandClient())
ctx := context.TODO()
_, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar1"), clientv3.OpPut("foo", "bar2")).Commit()
if err != rpctypes.ErrDuplicateKey {
t.Fatalf("expected %v, got %v", rpctypes.ErrDuplicateKey, err)
}
ops := make([]clientv3.Op, v3rpc.MaxOpsPerTxn+10)
for i := range ops {
ops[i] = clientv3.OpPut(fmt.Sprintf("foo%d", i), "")
}
_, err = kv.Txn(ctx).Then(ops...).Commit()
if err != rpctypes.ErrTooManyOps {
t.Fatalf("expected %v, got %v", rpctypes.ErrTooManyOps, err)
}
}
// Lock locks resources on sync
// This call blocks until you can get lock
func (s *Sync) Lock(path string, block bool) error {
for {
if s.HasLock(path) {
return nil
}
var err error
lease, err := s.etcdClient.Grant(s.withTimeout(), masterTTL)
var resp *etcd.TxnResponse
if err == nil {
cmp := etcd.Compare(etcd.CreateRevision(path), "=", 0)
put := etcd.OpPut(path, s.processID, etcd.WithLease(lease.ID))
resp, err = s.etcdClient.Txn(s.withTimeout()).If(cmp).Then(put).Commit()
}
if err != nil || !resp.Succeeded {
msg := fmt.Sprintf("failed to lock path %s", path)
if err != nil {
msg = fmt.Sprintf("failed to lock path %s: %s", path, err)
}
log.Notice(msg)
s.locks.Remove(path)
if !block {
return errors.New(msg)
}
time.Sleep(masterTTL * time.Second)
continue
}
log.Info("Locked %s", path)
s.locks.Set(path, lease.ID)
//Refresh master token
go func() {
defer func() {
log.Notice("releasing keepalive lock for %s", path)
s.locks.Remove(path)
}()
for s.HasLock(path) {
ch, err := s.etcdClient.KeepAlive(s.withTimeout(), lease.ID)
if err != nil {
log.Notice("failed to keepalive lock for %s %s", path, err)
return
}
for range ch {
}
}
}()
return nil
}
}
func NewUniqueKV(ctx context.Context, kv v3.KV, pfx, val string, opts ...v3.OpOption) (string, int64, error) {
for {
newKey := fmt.Sprintf("%s/%v", pfx, time.Now().UnixNano())
put := v3.OpPut(newKey, val, opts...)
cmp := v3.Compare(v3.ModifiedRevision(newKey), "=", 0)
resp, err := kv.Txn(ctx).If(cmp).Then(put).Commit()
if err != nil {
return "", 0, err
}
if !resp.Succeeded {
continue
}
return newKey, resp.Header.Revision, nil
}
}
func TestTxnWriteFail(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.Client(0))
clus.Members[0].Stop(t)
txnc, getc := make(chan struct{}), make(chan struct{})
go func() {
ctx, cancel := context.WithTimeout(context.TODO(), time.Second)
defer cancel()
resp, err := kv.Txn(ctx).Then(clientv3.OpPut("foo", "bar")).Commit()
if err == nil {
t.Fatalf("expected error, got response %v", resp)
}
close(txnc)
}()
go func() {
select {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for txn fail")
case <-txnc:
}
// and ensure the put didn't take
gresp, gerr := clus.Client(1).Get(context.TODO(), "foo")
if gerr != nil {
t.Fatal(gerr)
}
if len(gresp.Kvs) != 0 {
t.Fatalf("expected no keys, got %v", gresp.Kvs)
}
close(getc)
}()
select {
case <-time.After(5 * time.Second):
t.Fatalf("timed out waiting for get")
case <-getc:
}
// reconnect so terminate doesn't complain about double-close
clus.Members[0].Restart(t)
}
// commit attempts to apply the txn's changes to the server.
func (s *STM) commit() (ok bool, rr error) {
// read set must not change
cmps := make([]v3.Cmp, 0, len(s.rset))
for k, rk := range s.rset {
// use < to support updating keys that don't exist yet
cmp := v3.Compare(v3.ModifiedRevision(k), "<", rk.Revision()+1)
cmps = append(cmps, cmp)
}
// apply all writes
puts := make([]v3.Op, 0, len(s.wset))
for k, v := range s.wset {
puts = append(puts, v3.OpPut(k, v))
}
txnresp, err := s.client.Txn(context.TODO()).If(cmps...).Then(puts...).Commit()
return txnresp.Succeeded, err
}
// Proclaim lets the leader announce a new value without another election.
func (e *Election) Proclaim(ctx context.Context, val string) error {
if e.leaderSession == nil {
return ErrElectionNotLeader
}
cmp := v3.Compare(v3.CreateRevision(e.leaderKey), "=", e.leaderRev)
txn := e.client.Txn(ctx).If(cmp)
txn = txn.Then(v3.OpPut(e.leaderKey, val, v3.WithLease(e.leaderSession.Lease())))
tresp, terr := txn.Commit()
if terr != nil {
return terr
}
if !tresp.Succeeded {
e.leaderKey = ""
return ErrElectionNotLeader
}
return nil
}
func cas(key, value, newValue string, kvc clientv3.KV) error {
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
tnxRes, err := kvc.Txn(ctx).
If(clientv3.Compare(clientv3.Value(key), "=", value)).
Then(clientv3.OpPut(key, newValue)).
Else(clientv3.OpGet(key)).
Commit()
cancel()
if err != nil {
return err
}
if tnxRes.Succeeded {
return nil
}
log.Println(string(tnxRes.Responses[0].GetResponseRange().Kvs[0].Value))
return errors.New("release error")
}
func (b *blockEtcd) SaveSnapshot(name string) error {
vid := uint64(b.vid)
for {
sshotKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name)
inoKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(sshotKey), "=", 0),
).Then(
etcdv3.OpGet(inoKey),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
v := resp.Responses[0].GetResponseRange().Kvs[0]
inode := Snapshot{
Name: name,
When: time.Now(),
INodeRef: v.Value,
}
bytes, err := json.Marshal(inode)
if err != nil {
return err
}
tx = b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(inoKey), "=", v.Version),
).Then(
etcdv3.OpPut(sshotKey, string(bytes)),
)
resp, err = tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
continue
}
return nil
}
}
func (b *blockEtcd) Lock(lease int64) error {
if lease == 0 {
return torus.ErrInvalid
}
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(uint64(b.vid)), "blocklock")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), "=", 0),
).Then(
etcdv3.OpPut(k, b.Etcd.UUID(), etcdv3.WithLease(etcdv3.LeaseID(lease))),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func (b *blockEtcd) SyncINode(inode torus.INodeRef) error {
vid := uint64(inode.Volume())
inodeBytes := string(inode.ToBytes())
k := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blocklock")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(k), ">", 0),
etcdv3.Compare(etcdv3.Value(k), "=", b.Etcd.UUID()),
).Then(
etcdv3.OpPut(etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode"), inodeBytes),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrLocked
}
return nil
}
func TestTxnSuccess(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.Client(0))
_, err := kv.Txn().Then(clientv3.OpPut("foo", "bar", 0)).Commit()
if err != nil {
t.Fatal(err)
}
resp, err := kv.Get("foo", 0)
if err != nil {
t.Fatal(err)
}
if len(resp.Kvs) != 1 || string(resp.Kvs[0].Key) != "foo" {
t.Fatalf("unexpected Get response %v", resp)
}
}
// Campaign puts a value as eligible for the election. It blocks until
// it is elected, an error occurs, or the context is cancelled.
func (e *Election) Campaign(ctx context.Context, val string) error {
s, serr := NewSession(e.client)
if serr != nil {
return serr
}
k := fmt.Sprintf("%s/%x", e.keyPrefix, s.Lease())
txn := e.client.Txn(ctx).If(v3.Compare(v3.CreateRevision(k), "=", 0))
txn = txn.Then(v3.OpPut(k, val, v3.WithLease(s.Lease())))
txn = txn.Else(v3.OpGet(k))
resp, err := txn.Commit()
if err != nil {
return err
}
e.leaderKey, e.leaderRev, e.leaderSession = k, resp.Header.Revision, s
if !resp.Succeeded {
kv := resp.Responses[0].GetResponseRange().Kvs[0]
e.leaderRev = kv.CreateRevision
if string(kv.Value) != val {
if err = e.Proclaim(ctx, val); err != nil {
e.Resign(ctx)
return err
}
}
}
err = waitDeletes(ctx, e.client, e.keyPrefix, v3.WithPrefix(), v3.WithRev(e.leaderRev-1))
if err != nil {
// clean up in case of context cancel
select {
case <-ctx.Done():
e.Resign(e.client.Ctx())
default:
e.leaderSession = nil
}
return err
}
return nil
}
func (c *etcdCtx) AtomicModifyKey(k []byte, f AtomicModifyFunc) (interface{}, error) {
key := string(k)
resp, err := c.etcd.Client.Get(c.getContext(), key)
if err != nil {
return nil, err
}
var version int64
var value []byte
if len(resp.Kvs) != 1 {
version = 0
value = []byte{}
} else {
kv := resp.Kvs[0]
version = kv.Version
value = kv.Value
}
for {
newBytes, fval, err := f(value)
if err != nil {
return nil, err
}
txn := c.etcd.Client.Txn(c.getContext()).If(
etcdv3.Compare(etcdv3.Version(key), "=", version),
).Then(
etcdv3.OpPut(key, string(newBytes)),
).Else(
etcdv3.OpGet(key),
)
resp, err := txn.Commit()
if err != nil {
return nil, err
}
if resp.Succeeded {
return fval, nil
}
promAtomicRetries.WithLabelValues(string(key)).Inc()
kv := resp.Responses[0].GetResponseRange().Kvs[0]
version = kv.Version
value = kv.Value
}
}
