// compact compacts etcd store and returns current rev.
// It will return the current compact time and global revision if no error occurred.
// Note that CAS fail will not incur any error.
func compact(ctx context.Context, client *clientv3.Client, t, rev int64) (int64, int64, error) {
resp, err := client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.Version(compactRevKey), "=", t),
).Then(
clientv3.OpPut(compactRevKey, strconv.FormatInt(rev, 10)), // Expect side effect: increment Version
).Else(
clientv3.OpGet(compactRevKey),
).Commit()
if err != nil {
return t, rev, err
}
curRev := resp.Header.Revision
if !resp.Succeeded {
curTime := resp.Responses[0].GetResponseRange().Kvs[0].Version
return curTime, curRev, nil
}
curTime := t + 1
if rev == 0 {
// We don't compact on bootstrap.
return curTime, curRev, nil
}
if _, err = client.Compact(ctx, rev); err != nil {
return curTime, curRev, err
}
glog.Infof("etcd: compacted rev (%d), endpoints (%v)", rev, client.Endpoints())
return curTime, curRev, nil
}
func (s *stmSerializable) gets() (keys []string, ops []v3.Op) {
for k := range s.rset {
keys = append(keys, k)
ops = append(ops, v3.OpGet(k))
}
return
}
func rangeFunc(cmd *cobra.Command, args []string) {
if len(args) == 0 || len(args) > 2 {
fmt.Fprintln(os.Stderr, cmd.Usage())
os.Exit(1)
}
k := args[0]
end := ""
if len(args) == 2 {
end = args[1]
}
if rangeConsistency == "l" {
fmt.Println("bench with linearizable range")
} else if rangeConsistency == "s" {
fmt.Println("bench with serializable range")
} else {
fmt.Fprintln(os.Stderr, cmd.Usage())
os.Exit(1)
}
requests := make(chan v3.Op, totalClients)
clients := mustCreateClients(totalClients, totalConns)
bar = pb.New(rangeTotal)
bar.Format("Bom !")
bar.Start()
r := newReport()
for i := range clients {
wg.Add(1)
go func(c *v3.Client) {
defer wg.Done()
for op := range requests {
st := time.Now()
_, err := c.Do(context.Background(), op)
r.Results() <- report.Result{Err: err, Start: st, End: time.Now()}
bar.Increment()
}
}(clients[i])
}
go func() {
for i := 0; i < rangeTotal; i++ {
opts := []v3.OpOption{v3.WithRange(end)}
if rangeConsistency == "s" {
opts = append(opts, v3.WithSerializable())
}
op := v3.OpGet(k, opts...)
requests <- op
}
close(requests)
}()
rc := r.Run()
wg.Wait()
close(r.Results())
bar.Finish()
fmt.Printf("%s", <-rc)
}
// Lock locks the mutex with a cancellable context. If the context is cancelled
// while trying to acquire the lock, the mutex tries to clean its stale lock entry.
func (m *Mutex) Lock(ctx context.Context) error {
s, serr := NewSession(m.client)
if serr != nil {
return serr
}
m.myKey = fmt.Sprintf("%s/%x", m.pfx, s.Lease())
cmp := v3.Compare(v3.CreateRevision(m.myKey), "=", 0)
// put self in lock waiters via myKey; oldest waiter holds lock
put := v3.OpPut(m.myKey, "", v3.WithLease(s.Lease()))
// reuse key in case this session already holds the lock
get := v3.OpGet(m.myKey)
resp, err := m.client.Txn(ctx).If(cmp).Then(put).Else(get).Commit()
if err != nil {
return err
}
m.myRev = resp.Header.Revision
if !resp.Succeeded {
m.myRev = resp.Responses[0].GetResponseRange().Kvs[0].CreateRevision
}
// wait for deletion revisions prior to myKey
err = waitDeletes(ctx, m.client, m.pfx, v3.WithPrefix(), v3.WithRev(m.myRev-1))
// release lock key if cancelled
select {
case <-ctx.Done():
m.Unlock()
default:
}
return err
}
func TestTxnReadRetry(t *testing.T) {
defer testutil.AfterTest(t)
clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 3})
defer clus.Terminate(t)
kv := clientv3.NewKV(clus.Client(0))
clus.Members[0].Stop(t)
<-clus.Members[0].StopNotify()
donec := make(chan struct{})
go func() {
ctx := context.TODO()
_, err := kv.Txn(ctx).Then(clientv3.OpGet("foo")).Commit()
if err != nil {
t.Fatalf("expected response, got error %v", err)
}
donec <- struct{}{}
}()
// wait for txn to fail on disconnect
time.Sleep(100 * time.Millisecond)
// restart node; client should resume
clus.Members[0].Restart(t)
select {
case <-donec:
case <-time.After(5 * time.Second):
t.Fatalf("waited too long")
}
}
func parseRequestUnion(line string) (*clientv3.Op, error) {
args := argify(line)
if len(args) < 2 {
return nil, fmt.Errorf("invalid txn compare request: %s", line)
}
opc := make(chan clientv3.Op, 1)
put := NewPutCommand()
put.Run = func(cmd *cobra.Command, args []string) {
key, value, opts := getPutOp(cmd, args)
opc <- clientv3.OpPut(key, value, opts...)
}
get := NewGetCommand()
get.Run = func(cmd *cobra.Command, args []string) {
key, opts := getGetOp(cmd, args)
opc <- clientv3.OpGet(key, opts...)
}
del := NewDelCommand()
del.Run = func(cmd *cobra.Command, args []string) {
key, opts := getDelOp(cmd, args)
opc <- clientv3.OpDelete(key, opts...)
}
cmds := &cobra.Command{SilenceErrors: true}
cmds.AddCommand(put, get, del)
cmds.SetArgs(args)
if err := cmds.Execute(); err != nil {
return nil, fmt.Errorf("invalid txn request: %s", line)
}
op := <-opc
return &op, nil
}
func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.Object, v reflect.Value, precondtions *storage.Preconditions) error {
getResp, err := s.client.KV.Get(ctx, key)
if err != nil {
return err
}
for {
origState, err := s.getState(getResp, key, v, false)
if err != nil {
return err
}
if err := checkPreconditions(key, precondtions, origState.obj); err != nil {
return err
}
txnResp, err := s.client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
).Then(
clientv3.OpDelete(key),
).Else(
clientv3.OpGet(key),
).Commit()
if err != nil {
return err
}
if !txnResp.Succeeded {
getResp = (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
glog.V(4).Infof("deletion of %s failed because of a conflict, going to retry", key)
continue
}
return decode(s.codec, s.versioner, origState.data, out, origState.rev)
}
}
func (s *stmSerializable) gets() ([]string, []v3.Op) {
keys := make([]string, 0, len(s.rset))
ops := make([]v3.Op, 0, len(s.rset))
for k := range s.rset {
keys = append(keys, k)
ops = append(ops, v3.OpGet(k))
}
return keys, ops
}
// GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
func (s *store) GuaranteedUpdate(ctx context.Context, key string, out runtime.Object, ignoreNotFound bool, precondtions *storage.Preconditions, tryUpdate storage.UpdateFunc) error {
v, err := conversion.EnforcePtr(out)
if err != nil {
panic("unable to convert output object to pointer")
}
key = keyWithPrefix(s.pathPrefix, key)
getResp, err := s.client.KV.Get(ctx, key)
if err != nil {
return err
}
for {
origState, err := s.getState(getResp, key, v, ignoreNotFound)
if err != nil {
return err
}
if err := checkPreconditions(key, precondtions, origState.obj); err != nil {
return err
}
ret, ttl, err := s.updateState(origState, tryUpdate)
if err != nil {
return err
}
data, err := runtime.Encode(s.codec, ret)
if err != nil {
return err
}
if bytes.Equal(data, origState.data) {
return decode(s.codec, s.versioner, origState.data, out, origState.rev)
}
opts, err := s.ttlOpts(ctx, int64(ttl))
if err != nil {
return err
}
txnResp, err := s.client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
).Then(
clientv3.OpPut(key, string(data), opts...),
).Else(
clientv3.OpGet(key),
).Commit()
if err != nil {
return err
}
if !txnResp.Succeeded {
getResp = (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
glog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key)
continue
}
putResp := txnResp.Responses[0].GetResponsePut()
return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
}
}
func rangeFunc(cmd *cobra.Command, args []string) {
if len(args) == 0 || len(args) > 2 {
fmt.Fprintln(os.Stderr, cmd.Usage())
os.Exit(1)
}
k := args[0]
end := ""
if len(args) == 2 {
end = args[1]
}
if rangeConsistency == "l" {
fmt.Println("bench with linearizable range")
} else if rangeConsistency == "s" {
fmt.Println("bench with serializable range")
} else {
fmt.Fprintln(os.Stderr, cmd.Usage())
os.Exit(1)
}
results = make(chan result)
requests := make(chan v3.Op, totalClients)
bar = pb.New(rangeTotal)
clients := mustCreateClients(totalClients, totalConns)
bar.Format("Bom !")
bar.Start()
for i := range clients {
wg.Add(1)
go doRange(clients[i].KV, requests)
}
pdoneC := printReport(results)
go func() {
for i := 0; i < rangeTotal; i++ {
opts := []v3.OpOption{v3.WithRange(end)}
if rangeConsistency == "s" {
opts = append(opts, v3.WithSerializable())
}
op := v3.OpGet(k, opts...)
requests <- op
}
close(requests)
}()
wg.Wait()
bar.Finish()
close(results)
<-pdoneC
}
func (c *etcdCtx) GetVolumes() ([]*models.Volume, torus.VolumeID, error) {
promOps.WithLabelValues("get-volumes").Inc()
txn := c.etcd.Client.Txn(c.getContext()).Then(
etcdv3.OpGet(MkKey("meta", "volumeminter")),
etcdv3.OpGet(MkKey("volumeid"), etcdv3.WithPrefix()),
)
resp, err := txn.Commit()
if err != nil {
return nil, 0, err
}
highwater := BytesToUint64(resp.Responses[0].GetResponseRange().Kvs[0].Value)
list := resp.Responses[1].GetResponseRange().Kvs
var out []*models.Volume
for _, x := range list {
v := &models.Volume{}
err := v.Unmarshal(x.Value)
if err != nil {
return nil, 0, err
}
out = append(out, v)
}
return out, torus.VolumeID(highwater), nil
}
func (s *store) unconditionalDelete(ctx context.Context, key string, out runtime.Object) error {
// We need to do get and delete in single transaction in order to
// know the value and revision before deleting it.
txnResp, err := s.client.KV.Txn(ctx).If().Then(
clientv3.OpGet(key),
clientv3.OpDelete(key),
).Commit()
if err != nil {
return err
}
getResp := txnResp.Responses[0].GetResponseRange()
if len(getResp.Kvs) == 0 {
return storage.NewKeyNotFoundError(key, 0)
}
kv := getResp.Kvs[0]
return decode(s.codec, s.versioner, kv.Value, out, kv.ModRevision)
}
func RangeRequestToOp(r *pb.RangeRequest) clientv3.Op {
opts := []clientv3.OpOption{}
if len(r.RangeEnd) != 0 {
opts = append(opts, clientv3.WithRange(string(r.RangeEnd)))
}
opts = append(opts, clientv3.WithRev(r.Revision))
opts = append(opts, clientv3.WithLimit(r.Limit))
opts = append(opts, clientv3.WithSort(
clientv3.SortTarget(r.SortTarget),
clientv3.SortOrder(r.SortOrder)),
)
if r.Serializable {
opts = append(opts, clientv3.WithSerializable())
}
return clientv3.OpGet(string(r.Key), opts...)
}
func (b *blockEtcd) SaveSnapshot(name string) error {
vid := uint64(b.vid)
for {
sshotKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "snapshots", name)
inoKey := etcd.MkKey("volumemeta", etcd.Uint64ToHex(vid), "blockinode")
tx := b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(sshotKey), "=", 0),
).Then(
etcdv3.OpGet(inoKey),
)
resp, err := tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrExists
}
v := resp.Responses[0].GetResponseRange().Kvs[0]
inode := Snapshot{
Name: name,
When: time.Now(),
INodeRef: v.Value,
}
bytes, err := json.Marshal(inode)
if err != nil {
return err
}
tx = b.Etcd.Client.Txn(b.getContext()).If(
etcdv3.Compare(etcdv3.Version(inoKey), "=", v.Version),
).Then(
etcdv3.OpPut(sshotKey, string(bytes)),
)
resp, err = tx.Commit()
if err != nil {
return err
}
if !resp.Succeeded {
continue
}
return nil
}
}
func cas(key, value, newValue string, kvc clientv3.KV) error {
ctx, cancel := context.WithTimeout(context.Background(), requestTimeout)
tnxRes, err := kvc.Txn(ctx).
If(clientv3.Compare(clientv3.Value(key), "=", value)).
Then(clientv3.OpPut(key, newValue)).
Else(clientv3.OpGet(key)).
Commit()
cancel()
if err != nil {
return err
}
if tnxRes.Succeeded {
return nil
}
log.Println(string(tnxRes.Responses[0].GetResponseRange().Kvs[0].Value))
return errors.New("release error")
}
func (c *etcdCtx) AtomicModifyKey(k []byte, f AtomicModifyFunc) (interface{}, error) {
key := string(k)
resp, err := c.etcd.Client.Get(c.getContext(), key)
if err != nil {
return nil, err
}
var version int64
var value []byte
if len(resp.Kvs) != 1 {
version = 0
value = []byte{}
} else {
kv := resp.Kvs[0]
version = kv.Version
value = kv.Value
}
for {
newBytes, fval, err := f(value)
if err != nil {
return nil, err
}
txn := c.etcd.Client.Txn(c.getContext()).If(
etcdv3.Compare(etcdv3.Version(key), "=", version),
).Then(
etcdv3.OpPut(key, string(newBytes)),
).Else(
etcdv3.OpGet(key),
)
resp, err := txn.Commit()
if err != nil {
return nil, err
}
if resp.Succeeded {
return fval, nil
}
promAtomicRetries.WithLabelValues(string(key)).Inc()
kv := resp.Responses[0].GetResponseRange().Kvs[0]
version = kv.Version
value = kv.Value
}
}
// Campaign puts a value as eligible for the election. It blocks until
// it is elected, an error occurs, or the context is cancelled.
func (e *Election) Campaign(ctx context.Context, val string) error {
s, serr := NewSession(e.client)
if serr != nil {
return serr
}
k := fmt.Sprintf("%s/%x", e.keyPrefix, s.Lease())
txn := e.client.Txn(ctx).If(v3.Compare(v3.CreateRevision(k), "=", 0))
txn = txn.Then(v3.OpPut(k, val, v3.WithLease(s.Lease())))
txn = txn.Else(v3.OpGet(k))
resp, err := txn.Commit()
if err != nil {
return err
}
e.leaderKey, e.leaderRev, e.leaderSession = k, resp.Header.Revision, s
if !resp.Succeeded {
kv := resp.Responses[0].GetResponseRange().Kvs[0]
e.leaderRev = kv.CreateRevision
if string(kv.Value) != val {
if err = e.Proclaim(ctx, val); err != nil {
e.Resign(ctx)
return err
}
}
}
err = waitDeletes(ctx, e.client, e.keyPrefix, v3.WithPrefix(), v3.WithRev(e.leaderRev-1))
if err != nil {
// clean up in case of context cancel
select {
case <-ctx.Done():
e.Resign(e.client.Ctx())
default:
e.leaderSession = nil
}
return err
}
return nil
}
func ExampleKV_do() {
cli, err := clientv3.New(clientv3.Config{
Endpoints: endpoints,
DialTimeout: dialTimeout,
})
if err != nil {
log.Fatal(err)
}
defer cli.Close()
ops := []clientv3.Op{
clientv3.OpPut("put-key", "123"),
clientv3.OpGet("put-key"),
clientv3.OpPut("put-key", "456")}
for _, op := range ops {
if _, err := cli.Do(context.TODO(), op); err != nil {
log.Fatal(err)
}
}
}
func (e *Etcd) getGlobalMetadata() error {
txn := e.Client.Txn(context.Background())
resp, err := txn.If(
etcdv3.Compare(etcdv3.Version(MkKey("meta", "globalmetadata")), ">", 0),
).Then(
etcdv3.OpGet(MkKey("meta", "globalmetadata")),
).Commit()
if err != nil {
return err
}
if !resp.Succeeded {
return torus.ErrNoGlobalMetadata
}
var gmd torus.GlobalMetadata
err = json.Unmarshal(resp.Responses[0].GetResponseRange().Kvs[0].Value, &gmd)
if err != nil {
return err
}
e.global = gmd
return nil
}
// GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
func (s *store) GuaranteedUpdate(
ctx context.Context, key string, out runtime.Object, ignoreNotFound bool,
precondtions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error {
trace := util.NewTrace(fmt.Sprintf("GuaranteedUpdate etcd3: %s", reflect.TypeOf(out).String()))
defer trace.LogIfLong(500 * time.Millisecond)
v, err := conversion.EnforcePtr(out)
if err != nil {
panic("unable to convert output object to pointer")
}
key = keyWithPrefix(s.pathPrefix, key)
var origState *objState
if len(suggestion) == 1 && suggestion[0] != nil {
origState, err = s.getStateFromObject(suggestion[0])
if err != nil {
return err
}
} else {
getResp, err := s.client.KV.Get(ctx, key, s.getOps...)
if err != nil {
return err
}
origState, err = s.getState(getResp, key, v, ignoreNotFound)
if err != nil {
return err
}
}
trace.Step("initial value restored")
for {
if err := checkPreconditions(key, precondtions, origState.obj); err != nil {
return err
}
ret, ttl, err := s.updateState(origState, tryUpdate)
if err != nil {
return err
}
data, err := runtime.Encode(s.codec, ret)
if err != nil {
return err
}
if bytes.Equal(data, origState.data) {
return decode(s.codec, s.versioner, origState.data, out, origState.rev)
}
opts, err := s.ttlOpts(ctx, int64(ttl))
if err != nil {
return err
}
trace.Step("Transaction prepared")
txnResp, err := s.client.KV.Txn(ctx).If(
clientv3.Compare(clientv3.ModRevision(key), "=", origState.rev),
).Then(
clientv3.OpPut(key, string(data), opts...),
).Else(
clientv3.OpGet(key),
).Commit()
if err != nil {
return err
}
trace.Step("Transaction committed")
if !txnResp.Succeeded {
getResp := (*clientv3.GetResponse)(txnResp.Responses[0].GetResponseRange())
glog.V(4).Infof("GuaranteedUpdate of %s failed because of a conflict, going to retry", key)
origState, err = s.getState(getResp, key, v, ignoreNotFound)
if err != nil {
return err
}
trace.Step("Retry value restored")
continue
}
putResp := txnResp.Responses[0].GetResponsePut()
return decode(s.codec, s.versioner, data, out, putResp.Header.Revision)
}
}
