// checkConsistency stops the cluster for a moment and get the hashes of KV storages. func (c *cluster) checkConsistency() error { hashes := make(map[string]uint32) for _, u := range c.GRPCURLs { conn, err := grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return err } kvc := pb.NewKVClient(conn) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) resp, err := kvc.Hash(ctx, &pb.HashRequest{}) hv := resp.Hash if resp != nil && err != nil { return err } cancel() hashes[u] = hv } if !checkConsistency(hashes) { return fmt.Errorf("check consistency fails: %v", hashes) } return nil }
// NewClientV3 creates a new grpc client connection to the member func NewClientV3(m *member) (*clientv3.Client, error) { if m.grpcAddr == "" { return nil, fmt.Errorf("member not configured for grpc") } f := func(a string, t time.Duration) (net.Conn, error) { return net.Dial("unix", a) } unixdialer := grpc.WithDialer(f) opts := []grpc.DialOption{ unixdialer, grpc.WithBlock(), grpc.WithTimeout(5 * time.Second)} if m.ClientTLSInfo != nil { tlscfg, err := m.ClientTLSInfo.ClientConfig() if err != nil { return nil, err } creds := credentials.NewTLS(tlscfg) opts = append(opts, grpc.WithTransportCredentials(creds)) } else { opts = append(opts, grpc.WithInsecure()) } conn, err := grpc.Dial(m.grpcAddr, opts...) if err != nil { return nil, err } return clientv3.NewFromConn(conn), nil }
// Dial establishes a connection for a given endpoint using the client's config func (c *Client) Dial(endpoint string) (*grpc.ClientConn, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithTimeout(c.cfg.DialTimeout), } if c.creds != nil { opts = append(opts, grpc.WithTransportCredentials(*c.creds)) } else { opts = append(opts, grpc.WithInsecure()) } if url, uerr := url.Parse(endpoint); uerr == nil && url.Scheme == "unix" { f := func(a string, t time.Duration) (net.Conn, error) { return net.DialTimeout("unix", a, t) } // strip unix:// prefix so certs work endpoint = url.Host opts = append(opts, grpc.WithDialer(f)) } conn, err := grpc.Dial(endpoint, opts...) if err != nil { return nil, err } return conn, nil }
// Dial establishes a connection for a given endpoint using the client's config func (c *Client) Dial(endpoint string) (*grpc.ClientConn, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithTimeout(c.cfg.DialTimeout), } if c.creds != nil { opts = append(opts, grpc.WithTransportCredentials(*c.creds)) } else { opts = append(opts, grpc.WithInsecure()) } proto := "tcp" if url, uerr := url.Parse(endpoint); uerr == nil && url.Scheme == "unix" { proto = "unix" // strip unix:// prefix so certs work endpoint = url.Host } f := func(a string, t time.Duration) (net.Conn, error) { select { case <-c.ctx.Done(): return nil, c.ctx.Err() default: } return net.DialTimeout(proto, a, t) } opts = append(opts, grpc.WithDialer(f)) conn, err := grpc.Dial(endpoint, opts...) if err != nil { return nil, err } return conn, nil }
func TestDialTimeout(t *testing.T) { conn, err := grpc.Dial("Non-Existent.Server:80", grpc.WithTimeout(time.Millisecond)) if err == nil { conn.Close() } if err != grpc.ErrClientConnTimeout { t.Fatalf("grpc.Dial(_, _) = %v, %v, want %v", conn, err, grpc.ErrClientConnTimeout) } }
func (s *stresser) Stress() error { conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return fmt.Errorf("%v (%s)", err, s.Endpoint) } defer conn.Close() ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} wg.Add(s.N) s.mu.Lock() s.conn = conn s.cancel = cancel s.wg = wg s.mu.Unlock() kvc := pb.NewKVClient(conn) for i := 0; i < s.N; i++ { go func(i int) { defer wg.Done() for { // TODO: 10-second is enough timeout to cover leader failure // and immediate leader election. Find out what other cases this // could be timed out. putctx, putcancel := context.WithTimeout(ctx, 10*time.Second) _, err := kvc.Put(putctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))), Value: []byte(randStr(s.KeySize)), }) putcancel() if err != nil { if grpc.ErrorDesc(err) == context.DeadlineExceeded.Error() { // This retries when request is triggered at the same time as // leader failure. When we terminate the leader, the request to // that leader cannot be processed, and times out. Also requests // to followers cannot be forwarded to the old leader, so timing out // as well. We want to keep stressing until the cluster elects a // new leader and start processing requests again. continue } return } s.mu.Lock() s.success++ s.mu.Unlock() } }(i) } <-ctx.Done() return nil }
// Dial establishes a connection for a given endpoint using the client's config func (c *Client) Dial(endpoint string) (*grpc.ClientConn, error) { // TODO: enable grpc.WithTransportCredentials(creds) conn, err := grpc.Dial( endpoint, grpc.WithBlock(), grpc.WithTimeout(c.cfg.DialTimeout), grpc.WithInsecure()) if err != nil { return nil, err } return conn, nil }
// setHealthKey sets health key on all given urls. func setHealthKey(us []string) error { for _, u := range us { conn, err := grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return fmt.Errorf("%v (%s)", err, u) } ctx, cancel := context.WithTimeout(context.Background(), time.Second) kvc := pb.NewKVClient(conn) _, err = kvc.Put(ctx, &pb.PutRequest{Key: []byte("health"), Value: []byte("good")}) cancel() if err != nil { return err } } return nil }
// Dial establishes a connection for a given endpoint using the client's config func (c *Client) Dial(endpoint string) (*grpc.ClientConn, error) { opts := []grpc.DialOption{ grpc.WithBlock(), grpc.WithTimeout(c.cfg.DialTimeout), } if c.creds != nil { opts = append(opts, grpc.WithTransportCredentials(*c.creds)) } else { opts = append(opts, grpc.WithInsecure()) } conn, err := grpc.Dial(endpoint, opts...) if err != nil { return nil, err } return conn, nil }
func (c *cluster) getKVHash() (map[string]int64, error) { hashes := make(map[string]int64) for _, u := range c.GRPCURLs { conn, err := grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return nil, err } kvc := pb.NewKVClient(conn) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) resp, err := kvc.Hash(ctx, &pb.HashRequest{}) if resp != nil && err != nil { return nil, err } cancel() hashes[u] = int64(resp.Hash) } return hashes, nil }
func (c *cluster) getRevision() (map[string]int64, error) { revs := make(map[string]int64) for _, u := range c.GRPCURLs { conn, err := grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return nil, err } kvc := pb.NewKVClient(conn) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) resp, err := kvc.Range(ctx, &pb.RangeRequest{Key: []byte("foo")}) if err != nil { return nil, err } cancel() revs[u] = resp.Header.Revision } return revs, nil }
func TestReconnectTimeout(t *testing.T) { lis, err := net.Listen("tcp", ":0") if err != nil { t.Fatalf("Failed to listen: %v", err) } _, port, err := net.SplitHostPort(lis.Addr().String()) if err != nil { t.Fatalf("Failed to parse listener address: %v", err) } addr := "localhost:" + port conn, err := grpc.Dial(addr, grpc.WithTimeout(5*time.Second), grpc.WithBlock(), grpc.WithInsecure()) if err != nil { t.Fatalf("Failed to dial to the server %q: %v", addr, err) } // Close unaccepted connection (i.e., conn). lis.Close() tc := testpb.NewTestServiceClient(conn) waitC := make(chan struct{}) go func() { defer close(waitC) argSize := 271828 respSize := 314159 payload, err := newPayload(testpb.PayloadType_COMPRESSABLE, int32(argSize)) if err != nil { t.Fatal(err) } req := &testpb.SimpleRequest{ ResponseType: testpb.PayloadType_COMPRESSABLE.Enum(), ResponseSize: proto.Int32(int32(respSize)), Payload: payload, } if _, err := tc.UnaryCall(context.Background(), req); err == nil { t.Fatalf("TestService/UnaryCall(_, _) = _, <nil>, want _, non-nil") } }() // Block untill reconnect times out. <-waitC if err := conn.Close(); err != grpc.ErrClientConnClosing { t.Fatalf("%v.Close() = %v, want %v", conn, err, grpc.ErrClientConnClosing) } }
func (s *stresser) Stress() error { conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return fmt.Errorf("%v (%s)", err, s.Endpoint) } defer conn.Close() ctx, cancel := context.WithCancel(context.Background()) wg := &sync.WaitGroup{} wg.Add(s.N) s.mu.Lock() s.conn = conn s.cancel = cancel s.wg = wg s.mu.Unlock() kvc := pb.NewKVClient(conn) for i := 0; i < s.N; i++ { go func(i int) { defer wg.Done() for { putctx, putcancel := context.WithTimeout(ctx, 5*time.Second) _, err := kvc.Put(putctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))), Value: []byte(randStr(s.KeySize)), }) putcancel() if err != nil { return } s.mu.Lock() s.success++ s.mu.Unlock() } }(i) } <-ctx.Done() return nil }
func (c *cluster) compactKV(rev int64) error { var ( conn *grpc.ClientConn err error ) for _, u := range c.GRPCURLs { conn, err = grpc.Dial(u, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { continue } kvc := pb.NewKVClient(conn) ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) _, err = kvc.Compact(ctx, &pb.CompactionRequest{Revision: rev}) cancel() if err == nil { return nil } } return err }
func (s *stresser) Stress() error { conn, err := grpc.Dial(s.Endpoint, grpc.WithInsecure(), grpc.WithTimeout(5*time.Second)) if err != nil { return fmt.Errorf("%v (%s)", err, s.Endpoint) } ctx, cancel := context.WithCancel(context.Background()) s.mu.Lock() s.conn = conn s.cancel = cancel s.mu.Unlock() kvc := pb.NewKVClient(conn) for i := 0; i < s.N; i++ { go func(i int) { for { putctx, putcancel := context.WithTimeout(ctx, 5*time.Second) _, err := kvc.Put(putctx, &pb.PutRequest{ Key: []byte(fmt.Sprintf("foo%d", rand.Intn(s.KeySuffixRange))), Value: []byte(randStr(s.KeySize)), }) putcancel() if grpc.ErrorDesc(err) == context.Canceled.Error() { return } s.mu.Lock() if err != nil { s.failure++ } else { s.success++ } s.mu.Unlock() } }(i) } <-ctx.Done() return nil }
func TestTLSDialTimeout(t *testing.T) { creds, err := credentials.NewClientTLSFromFile(tlsDir+"ca.pem", "x.test.youtube.com") if err != nil { t.Fatalf("Failed to create credentials %v", err) } conn, err := grpc.Dial("Non-Existent.Server:80", grpc.WithTransportCredentials(creds), grpc.WithTimeout(time.Millisecond)) if err == nil { conn.Close() } if err != grpc.ErrClientConnTimeout { t.Fatalf("grpc.Dial(_, _) = %v, %v, want %v", conn, err, grpc.ErrClientConnTimeout) } }