Example #1
1
func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) error {
	total := int64(0)

	go func() {
		for {
			time.Sleep(30 * time.Second)
			fmt.Println(atomic.LoadInt64(&total))
		}
	}()

	// TODO: remove the prefix of the destination cluster?
	s := mirror.NewSyncer(c, mmprefix, 0)

	rc, errc := s.SyncBase(ctx)

	for r := range rc {
		for _, kv := range r.Kvs {
			_, err := dc.Put(ctx, string(kv.Key), string(kv.Value))
			if err != nil {
				return err
			}
			atomic.AddInt64(&total, 1)
		}
	}

	err := <-errc
	if err != nil {
		return err
	}

	wc := s.SyncUpdates(ctx)

	for wr := range wc {
		if wr.CompactRevision != 0 {
			return rpctypes.ErrCompacted
		}

		var rev int64
		ops := []clientv3.Op{}

		for _, ev := range wr.Events {
			nrev := ev.Kv.ModRevision
			if rev != 0 && nrev > rev {
				_, err := dc.Txn(ctx).Then(ops...).Commit()
				if err != nil {
					return err
				}
				ops = []clientv3.Op{}
			}
			switch ev.Type {
			case storagepb.PUT:
				ops = append(ops, clientv3.OpPut(string(ev.Kv.Key), string(ev.Kv.Value)))
				atomic.AddInt64(&total, 1)
			case storagepb.DELETE, storagepb.EXPIRE:
				ops = append(ops, clientv3.OpDelete(string(ev.Kv.Key)))
				atomic.AddInt64(&total, 1)
			default:
				panic("unexpected event type")
			}
		}

		if len(ops) != 0 {
			_, err := dc.Txn(ctx).Then(ops...).Commit()
			if err != nil {
				return err
			}
		}
	}

	return nil
}
Example #2
0
func TestMirrorSyncBase(t *testing.T) {
	cluster := integration.NewClusterV3(nil, &integration.ClusterConfig{Size: 1})
	defer cluster.Terminate(nil)

	cli := cluster.Client(0)
	ctx := context.TODO()

	keyCh := make(chan string)
	var wg sync.WaitGroup

	for i := 0; i < 50; i++ {
		wg.Add(1)

		go func() {
			defer wg.Done()

			for key := range keyCh {
				if _, err := cli.Put(ctx, key, "test"); err != nil {
					t.Fatal(err)
				}
			}
		}()
	}

	for i := 0; i < 2000; i++ {
		keyCh <- fmt.Sprintf("test%d", i)
	}

	close(keyCh)
	wg.Wait()

	syncer := mirror.NewSyncer(cli, "test", 0)
	respCh, errCh := syncer.SyncBase(ctx)

	count := 0

	for resp := range respCh {
		count = count + len(resp.Kvs)
		if !resp.More {
			break
		}
	}

	for err := range errCh {
		t.Fatalf("unexpected error %v", err)
	}

	if count != 2000 {
		t.Errorf("unexpected kv count: %d", count)
	}
}
Example #3
0
func TestMirrorSync(t *testing.T) {
	defer testutil.AfterTest(t)

	clus := integration.NewClusterV3(t, &integration.ClusterConfig{Size: 1})
	defer clus.Terminate(t)

	c := clus.Client(0)
	_, err := c.KV.Put(context.TODO(), "foo", "bar")
	if err != nil {
		t.Fatal(err)
	}

	syncer := mirror.NewSyncer(c, "", 0)
	gch, ech := syncer.SyncBase(context.TODO())
	wkvs := []*storagepb.KeyValue{{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 2, Version: 1}}

	for g := range gch {
		if !reflect.DeepEqual(g.Kvs, wkvs) {
			t.Fatalf("kv = %v, want %v", g.Kvs, wkvs)
		}
	}

	for e := range ech {
		t.Fatalf("unexpected error %v", e)
	}

	wch := syncer.SyncUpdates(context.TODO())

	_, err = c.KV.Put(context.TODO(), "foo", "bar")
	if err != nil {
		t.Fatal(err)
	}

	select {
	case r := <-wch:
		wkv := &storagepb.KeyValue{Key: []byte("foo"), Value: []byte("bar"), CreateRevision: 2, ModRevision: 3, Version: 2}
		if !reflect.DeepEqual(r.Events[0].Kv, wkv) {
			t.Fatalf("kv = %v, want %v", r.Events[0].Kv, wkv)
		}
	case <-time.After(time.Second):
		t.Fatal("failed to receive update in one second")
	}
}
Example #4
0
// snapshot reads all of a watcher; returns compaction revision if incomplete
// TODO: stabilize snapshot format
func snapshot(w io.Writer, c *clientv3.Client, rev int64) int64 {
	s := mirror.NewSyncer(c, "", rev)

	rc, errc := s.SyncBase(context.TODO())

	for r := range rc {
		for _, kv := range r.Kvs {
			fmt.Fprintln(w, kv)
		}
	}

	err := <-errc
	if err != nil {
		if err == v3rpc.ErrCompacted {
			// will get correct compact revision on retry
			return rev + 1
		}
		// failed for some unknown reason, retry on same revision
		return rev
	}

	wc := s.SyncUpdates(context.TODO())

	for wr := range wc {
		if wr.Err() != nil {
			return wr.CompactRevision
		}
		for _, ev := range wr.Events {
			fmt.Fprintln(w, ev)
		}
		rev := wr.Events[len(wr.Events)-1].Kv.ModRevision
		if rev >= wr.Header.Revision {
			break
		}
	}

	return 0
}
Example #5
0
func makeMirror(ctx context.Context, c *clientv3.Client, dc *clientv3.Client) error {
	total := int64(0)

	go func() {
		for {
			time.Sleep(30 * time.Second)
			fmt.Println(atomic.LoadInt64(&total))
		}
	}()

	s := mirror.NewSyncer(c, mmprefix, 0)

	rc, errc := s.SyncBase(ctx)

	// if destination prefix is specified and remove destination prefix is true return error
	if mmnodestprefix && len(mmdestprefix) > 0 {
		ExitWithError(ExitBadArgs, fmt.Errorf("`--dest-prefix` and `--no-dest-prefix` cannot be set at the same time, choose one."))
	}

	// if remove destination prefix is false and destination prefix is empty set the value of destination prefix same as prefix
	if !mmnodestprefix && len(mmdestprefix) == 0 {
		mmdestprefix = mmprefix
	}

	for r := range rc {
		for _, kv := range r.Kvs {
			_, err := dc.Put(ctx, modifyPrefix(string(kv.Key)), string(kv.Value))
			if err != nil {
				return err
			}
			atomic.AddInt64(&total, 1)
		}
	}

	err := <-errc
	if err != nil {
		return err
	}

	wc := s.SyncUpdates(ctx)

	for wr := range wc {
		if wr.CompactRevision != 0 {
			return rpctypes.ErrCompacted
		}

		var rev int64
		ops := []clientv3.Op{}

		for _, ev := range wr.Events {
			nrev := ev.Kv.ModRevision
			if rev != 0 && nrev > rev {
				_, err := dc.Txn(ctx).Then(ops...).Commit()
				if err != nil {
					return err
				}
				ops = []clientv3.Op{}
			}
			switch ev.Type {
			case mvccpb.PUT:
				ops = append(ops, clientv3.OpPut(modifyPrefix(string(ev.Kv.Key)), string(ev.Kv.Value)))
				atomic.AddInt64(&total, 1)
			case mvccpb.DELETE:
				ops = append(ops, clientv3.OpDelete(modifyPrefix(string(ev.Kv.Key))))
				atomic.AddInt64(&total, 1)
			default:
				panic("unexpected event type")
			}
		}

		if len(ops) != 0 {
			_, err := dc.Txn(ctx).Then(ops...).Commit()
			if err != nil {
				return err
			}
		}
	}

	return nil
}