Пример #1
0
func newHarnessInternal(url string, s stopper, t testing.TB) *Harness {
	replicaSet := ReplicaSet{
		Addrs:                   url,
		PortStart:               2000,
		PortEnd:                 3000,
		MaxConnections:          5,
		MinIdleConnections:      5,
		ServerIdleTimeout:       5 * time.Minute,
		ServerClosePoolSize:     5,
		ClientIdleTimeout:       5 * time.Minute,
		MaxPerClientConnections: 250,
		GetLastErrorTimeout:     5 * time.Minute,
		MessageTimeout:          time.Minute,
	}
	log := tLogger{TB: t}
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &stats.HookClient{}},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	ensure.Nil(t, startstop.Start(objects, &log))
	return &Harness{
		T:          t,
		Stopper:    s,
		ReplicaSet: &replicaSet,
		Graph:      &graph,
		Log:        &log,
	}
}
Пример #2
0
func NewAPIs() (*apis, error) {
	apis := &apis{}
	apis.nodeDefinitions = relay.NewNodeDefinitions(relay.NodeDefinitionsConfig{
		IDFetcher: func(id string, info graphql.ResolveInfo, ctx context.Context) (interface{}, error) {
			resolvedID := relay.FromGlobalID(id)
			switch resolvedID.Type {
			case "Search":
				return apis.SearchService.ByClientID(ctx, resolvedID.ID)
			case "Space":
				return apis.SpaceService.ByID(ctx, resolvedID.ID)
			case "User":
				return apis.UserService.ByID(ctx, resolvedID.ID)
			default:
				return nil, fmt.Errorf("Unknown node type %s", resolvedID.Type)
			}
		},
		TypeResolve: func(p graphql.ResolveTypeParams) *graphql.Object {
			switch p.Value.(type) {
			case *Search:
				return apis.SearchAPI.Type
			case *Space:
				return apis.SpaceAPI.Type
			case *Task:
				return apis.TaskAPI.Type
			case *User:
				return apis.UserAPI.Type
			}
			return nil
		},
	})

	graph := &inject.Graph{}
	err := graph.Provide(
		&inject.Object{
			Value: apis,
		},
		&inject.Object{
			Value: apis.nodeDefinitions.NodeInterface,
			Name:  "node",
		},
	)
	if err != nil {
		return nil, err
	}

	err = graph.Populate()
	if err != nil {
		return nil, err
	}

	err = startstop.Start(graph.Objects(), nil)
	if err != nil {
		return nil, err
	}

	return apis, nil
}
Пример #3
0
func Main() error {
	messageTimeout := flag.Duration("message_timeout", 2*time.Minute, "timeout for one message to be proxied")
	clientIdleTimeout := flag.Duration("client_idle_timeout", 60*time.Minute, "idle timeout for client connections")
	getLastErrorTimeout := flag.Duration("get_last_error_timeout", time.Minute, "timeout for getLastError pinning")
	maxConnections := flag.Uint("max_connections", 100, "maximum number of connections per mongo")
	portStart := flag.Int("port_start", 6000, "start of port range")
	portEnd := flag.Int("port_end", 6010, "end of port range")
	addrs := flag.String("addrs", "localhost:27017", "comma separated list of mongo addresses")

	flag.Parse()

	replicaSet := dvara.ReplicaSet{
		Addrs:               *addrs,
		PortStart:           *portStart,
		PortEnd:             *portEnd,
		MessageTimeout:      *messageTimeout,
		ClientIdleTimeout:   *clientIdleTimeout,
		GetLastErrorTimeout: *getLastErrorTimeout,
		MaxConnections:      *maxConnections,
	}

	var statsClient stats.HookClient
	var log stdLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &statsClient},
	)
	if err != nil {
		return err
	}
	if err := graph.Populate(); err != nil {
		return err
	}
	objects := graph.Objects()

	// Temporarily setup the metrics against a test registry.
	gregistry := gangliamr.NewTestRegistry()
	for _, o := range objects {
		if rmO, ok := o.Value.(registerMetrics); ok {
			rmO.RegisterMetrics(gregistry)
		}
	}
	if err := startstop.Start(objects, &log); err != nil {
		return err
	}
	defer startstop.Stop(objects, &log)

	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	signal.Stop(ch)
	return nil
}
Пример #4
0
func Main() error {
	messageTimeout := flag.Duration("message_timeout", 2*time.Minute, "timeout for one message to be proxied")
	clientIdleTimeout := flag.Duration("client_idle_timeout", 60*time.Minute, "idle timeout for client connections")
	serverIdleTimeout := flag.Duration("server_idle_timeout", 1*time.Hour, "idle timeout for  server connections")
	serverClosePoolSize := flag.Uint("server_close_pool_size", 100, "number of goroutines that will handle closing server connections")
	getLastErrorTimeout := flag.Duration("get_last_error_timeout", time.Minute, "timeout for getLastError pinning")
	maxPerClientConnections := flag.Uint("max_per_client_connections", 100, "maximum number of connections per client")
	maxConnections := flag.Uint("max_connections", 100, "maximum number of connections per mongo")
	portStart := flag.Int("port_start", 6000, "start of port range")
	portEnd := flag.Int("port_end", 6010, "end of port range")
	addrs := flag.String("addrs", "localhost:27017", "comma separated list of mongo addresses")

	flag.Parse()

	replicaSet := dvara.ReplicaSet{
		Addrs:                   *addrs,
		PortStart:               *portStart,
		PortEnd:                 *portEnd,
		MessageTimeout:          *messageTimeout,
		ClientIdleTimeout:       *clientIdleTimeout,
		ServerIdleTimeout:       *serverIdleTimeout,
		ServerClosePoolSize:     *serverClosePoolSize,
		GetLastErrorTimeout:     *getLastErrorTimeout,
		MaxConnections:          *maxConnections,
		MaxPerClientConnections: *maxPerClientConnections,
	}

	var statsClient stats.HookClient
	var log stdLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &statsClient},
	)
	if err != nil {
		return err
	}
	if err := graph.Populate(); err != nil {
		return err
	}
	objects := graph.Objects()

	if err := startstop.Start(objects, &log); err != nil {
		return err
	}
	defer startstop.Stop(objects, &log)

	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	signal.Stop(ch)
	return nil
}
Пример #5
0
func TestNoAddrsGiven(t *testing.T) {
	t.Parallel()
	replicaSet := ReplicaSet{MaxConnections: 1}
	var log nopLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &stats.HookClient{}},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	err = startstop.Start(objects, &log)
	if err != errNoAddrsGiven {
		t.Fatalf("did not get expected error, got: %s", err)
	}
}
Пример #6
0
func newHarnessInternal(url string, s stopper, t testing.TB) *Harness {
	replicaSet := ReplicaSet{
		Addrs:                   url,
		ListenAddr:              "",
		PortStart:               2000,
		PortEnd:                 3000,
		MaxConnections:          5,
		MinIdleConnections:      5,
		ServerIdleTimeout:       5 * time.Minute,
		ServerClosePoolSize:     5,
		ClientIdleTimeout:       5 * time.Minute,
		MaxPerClientConnections: 250,
		GetLastErrorTimeout:     5 * time.Minute,
		MessageTimeout:          5 * time.Second,
	}
	var log nopLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &stats.HookClient{}},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	gregistry := gangliamr.NewTestRegistry()
	for _, o := range objects {
		if rmO, ok := o.Value.(registerMetrics); ok {
			rmO.RegisterMetrics(gregistry)
		}
	}
	ensure.Nil(t, startstop.Start(objects, &log))
	return &Harness{
		T:          t,
		Stopper:    s,
		ReplicaSet: &replicaSet,
		Graph:      &graph,
	}
}
Пример #7
0
func TestSingleNodeWhenExpectingRS(t *testing.T) {
	t.Parallel()
	mgoserver := mgotest.NewStartedServer(t)
	defer mgoserver.Stop()
	replicaSet := ReplicaSet{
		Addrs:          fmt.Sprintf("127.0.0.1:%d,127.0.0.1:%d", mgoserver.Port, mgoserver.Port+1),
		MaxConnections: 1,
	}
	var log nopLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &stats.HookClient{}},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	err = startstop.Start(objects, &log)
	if err == nil || !strings.Contains(err.Error(), "was expecting it to be in a replica set") {
		t.Fatalf("did not get expected error, got: %s", err)
	}
}
Пример #8
0
func Main() error {
	addrs := flag.String("addrs", "localhost:27017", "comma separated list of mongo addresses")
	clientIdleTimeout := flag.Duration("client_idle_timeout", 60*time.Minute, "idle timeout for client connections")
	getLastErrorTimeout := flag.Duration("get_last_error_timeout", time.Minute, "timeout for getLastError pinning")
	listenAddr := flag.String("listen", "127.0.0.1", "address for listening, for example, 127.0.0.1 for reachable only from the same machine, or 0.0.0.0 for reachable from other machines")
	maxConnections := flag.Uint("max_connections", 100, "maximum number of connections per mongo")
	maxPerClientConnections := flag.Uint("max_per_client_connections", 1, "maximum number of connections from a single client")
	messageTimeout := flag.Duration("message_timeout", 2*time.Minute, "timeout for one message to be proxied")
	password := flag.String("password", "", "mongodb password")
	portEnd := flag.Int("port_end", 6010, "end of port range")
	portStart := flag.Int("port_start", 6000, "start of port range")
	serverClosePoolSize := flag.Uint("server_close_pool_size", 1, "number of goroutines that will handle closing server connections.")
	serverIdleTimeout := flag.Duration("server_idle_timeout", 60*time.Minute, "duration after which a server connection will be considered idle")
	username := flag.String("username", "", "mongo db username")
	verbose := flag.Bool("verbose", false, "Be really verbose")
	metricsAddress := flag.String("metrics", "127.0.0.1:8125", "UDP address to send metrics to datadog, default is 127.0.0.1:8125")
	replicaName := flag.String("replica_name", "", "Replica name, used in metrics and logging, default is empty")

	flag.Parse()
	statsClient := NewDataDogStatsDClient(*metricsAddress, *replicaName)

	replicaSet := dvara.ReplicaSet{
		Addrs:                   *addrs,
		ClientIdleTimeout:       *clientIdleTimeout,
		GetLastErrorTimeout:     *getLastErrorTimeout,
		ListenAddr:              *listenAddr,
		MaxConnections:          *maxConnections,
		MaxPerClientConnections: *maxPerClientConnections,
		MessageTimeout:          *messageTimeout,
		Password:                *password,
		PortEnd:                 *portEnd,
		PortStart:               *portStart,
		ServerClosePoolSize:     *serverClosePoolSize,
		ServerIdleTimeout:       *serverIdleTimeout,
		Username:                *username,
	}

	// Extra space in logger, as word boundary
	log := stdLogger{*replicaName + " ", *verbose}
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &statsClient},
	)
	if err != nil {
		return err
	}
	if err := graph.Populate(); err != nil {
		return err
	}
	objects := graph.Objects()

	// Temporarily setup the metrics against a test registry.
	gregistry := gangliamr.NewTestRegistry()
	for _, o := range objects {
		if rmO, ok := o.Value.(registerMetrics); ok {
			rmO.RegisterMetrics(gregistry)
		}
	}
	if err := startstop.Start(objects, &log); err != nil {
		return err
	}
	defer startstop.Stop(objects, &log)

	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	signal.Stop(ch)
	return nil
}
Пример #9
0
func Main() error {
	messageTimeout := flag.Duration("message_timeout", 2*time.Minute, "timeout for one message to be proxied")
	clientIdleTimeout := flag.Duration("client_idle_timeout", 60*time.Minute, "idle timeout for client connections")
	getLastErrorTimeout := flag.Duration("get_last_error_timeout", time.Minute, "timeout for getLastError pinning")
	maxConnections := flag.Uint("max_connections", 1024, "maximum number of connections per mongo")
	portStart := flag.Int("port_start", 6000, "start of port range")
	portEnd := flag.Int("port_end", 6010, "end of port range")
	addrs := flag.String("addrs", "localhost:27017", "comma separated list of mongo addresses")
	host := flag.String("host", "127.0.0.1", "host to listen")
	maxPerClientConnections := flag.Uint("max_per_client_connections", 1024, "maximum number of connections per client")
	minIdleConnections := flag.Uint("min_idle_connection", 64, "the number of idle server connections we'll keep around")
	serverIdleTimeout := flag.Duration("server_idle_timeout", 60*time.Minute, "idle timeout for server connections")
	serverClosePoolSize := flag.Uint("server_close_poolsize", 1024, "the number of goroutines that will handle closing server connections")
	debug = flag.Bool("debug", false, "debug mode")

	flag.Parse()

	replicaSet := dvara.ReplicaSet{
		Addrs:                   *addrs,
		HostListen:              *host,
		PortStart:               *portStart,
		PortEnd:                 *portEnd,
		MaxConnections:          *maxConnections,
		MinIdleConnections:      *minIdleConnections,
		ServerIdleTimeout:       *serverIdleTimeout,
		ServerClosePoolSize:     *serverClosePoolSize,
		ClientIdleTimeout:       *clientIdleTimeout,
		MaxPerClientConnections: *maxPerClientConnections,
		GetLastErrorTimeout:     *getLastErrorTimeout,
		MessageTimeout:          *messageTimeout,
	}

	var statsClient stats.HookClient
	var log stdLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &log},
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &statsClient},
	)
	if err != nil {
		return err
	}
	if err := graph.Populate(); err != nil {
		return err
	}
	objects := graph.Objects()

	// Temporarily setup the metrics against a test registry.
	gregistry := gangliamr.NewTestRegistry()
	for _, o := range objects {
		if rmO, ok := o.Value.(registerMetrics); ok {
			rmO.RegisterMetrics(gregistry)
		}
	}
	if err := startstop.Start(objects, &log); err != nil {
		return err
	}
	defer startstop.Stop(objects, &log)

	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	signal.Stop(ch)
	return nil
}
Пример #10
0
func TestProxyQuery(t *testing.T) {
	t.Parallel()
	var p ProxyQuery
	var log nopLogger
	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &fakeProxyMapper{}},
		&inject.Object{Value: &fakeReplicaStateCompare{}},
		&inject.Object{Value: &log},
		&inject.Object{Value: &p},
	)
	ensure.Nil(t, err)
	ensure.Nil(t, graph.Populate())
	objects := graph.Objects()
	gregistry := gangliamr.NewTestRegistry()
	for _, o := range objects {
		if rmO, ok := o.Value.(registerMetrics); ok {
			rmO.RegisterMetrics(gregistry)
		}
	}
	ensure.Nil(t, startstop.Start(objects, &log))
	defer startstop.Stop(objects, &log)

	cases := []struct {
		Name   string
		Header *messageHeader
		Client io.ReadWriter
		Error  string
	}{
		{
			Name:   "EOF while reading flags from client",
			Header: &messageHeader{},
			Client: new(bytes.Buffer),
			Error:  "EOF",
		},
		{
			Name:   "EOF while reading collection name",
			Header: &messageHeader{},
			Client: fakeReadWriter{
				Reader: bytes.NewReader(
					[]byte{0, 0, 0, 0}, // flags int32 before collection name
				),
			},
			Error: "EOF",
		},
		{
			Name:   "EOF while reading skip/return",
			Header: &messageHeader{},
			Client: fakeReadWriter{
				Reader: bytes.NewReader(
					append(
						[]byte{0, 0, 0, 0}, // flags int32 before collection name
						adminCollectionName...,
					),
				),
			},
			Error: "EOF",
		},
		{
			Name:   "EOF while reading query document",
			Header: &messageHeader{},
			Client: fakeReadWriter{
				Reader: io.MultiReader(
					bytes.NewReader([]byte{0, 0, 0, 0}), // flags int32 before collection name
					bytes.NewReader(adminCollectionName),
					bytes.NewReader(
						[]byte{
							0, 0, 0, 0, // numberToSkip int32
							0, 0, 0, 0, // numberToReturn int32
							1, // partial bson document length header
						}),
				),
			},
			Error: "EOF",
		},
		{
			Name:   "error while unmarshaling query document",
			Header: &messageHeader{},
			Client: fakeReadWriter{
				Reader: io.MultiReader(
					bytes.NewReader([]byte{0, 0, 0, 0}), // flags int32 before collection name
					bytes.NewReader(adminCollectionName),
					bytes.NewReader(
						[]byte{
							0, 0, 0, 0, // numberToSkip int32
							0, 0, 0, 0, // numberToReturn int32
							5, 0, 0, 0, // bson document length header
							1, // bson document
						}),
				),
			},
			Error: "Document is corrupted",
		},
	}

	for _, c := range cases {
		err := p.Proxy(c.Header, c.Client, nil, nil)
		if err == nil || !strings.Contains(err.Error(), c.Error) {
			t.Fatalf("did not find expected error for %s, instead found %s", c.Name, err)
		}
	}
}
Пример #11
0
func Main() error {
	addrs := flag.String("addrs", "localhost:27017", "comma separated list of mongo addresses")
	clientIdleTimeout := flag.Duration("client_idle_timeout", 60*time.Minute, "idle timeout for client connections")
	getLastErrorTimeout := flag.Duration("get_last_error_timeout", time.Minute, "timeout for getLastError pinning")
	listenAddr := flag.String("listen", "127.0.0.1", "address for listening, for example, 127.0.0.1 for reachable only from the same machine, or 0.0.0.0 for reachable from other machines")
	maxConnections := flag.Uint("max_connections", 100, "maximum number of connections per mongo")
	maxPerClientConnections := flag.Uint("max_per_client_connections", 100, "maximum number of connections from a single client")
	messageTimeout := flag.Duration("message_timeout", 2*time.Minute, "timeout for one message to be proxied")
	password := flag.String("password", "", "mongodb password")
	portEnd := flag.Int("port_end", 6010, "end of port range")
	portStart := flag.Int("port_start", 6000, "start of port range")
	serverClosePoolSize := flag.Uint("server_close_pool_size", 1, "number of goroutines that will handle closing server connections.")
	serverIdleTimeout := flag.Duration("server_idle_timeout", 60*time.Minute, "duration after which a server connection will be considered idle")
	username := flag.String("username", "", "mongo db username")
	metricsAddress := flag.String("metrics", "127.0.0.1:8125", "UDP address to send metrics to datadog, default is 127.0.0.1:8125")
	replicaName := flag.String("replica_name", "", "Replica name, used in metrics and logging, default is empty")
	replicaSetName := flag.String("replica_set_name", "", "Replica set name, used to filter hosts runnning other replica sets")
	healthCheckInterval := flag.Duration("healthcheckinterval", 5*time.Second, "How often to run the health check")
	failedHealthCheckThreshold := flag.Uint("failedhealthcheckthreshold", 3, "How many failed checks before a restart")

	flag.Parse()
	statsClient := NewDataDogStatsDClient(*metricsAddress, "replica:"+*replicaName)

	replicaSet := dvara.ReplicaSet{
		Addrs:                   *addrs,
		ClientIdleTimeout:       *clientIdleTimeout,
		GetLastErrorTimeout:     *getLastErrorTimeout,
		ListenAddr:              *listenAddr,
		MaxConnections:          *maxConnections,
		MaxPerClientConnections: *maxPerClientConnections,
		MessageTimeout:          *messageTimeout,
		Password:                *password,
		PortEnd:                 *portEnd,
		PortStart:               *portStart,
		ServerClosePoolSize:     *serverClosePoolSize,
		ServerIdleTimeout:       *serverIdleTimeout,
		Username:                *username,
		Name:                    *replicaSetName,
	}
	stateManager := dvara.NewStateManager(&replicaSet)

	// Actual logger
	corelog.SetupLogFmtLoggerTo(os.Stderr)
	corelog.SetStandardFields("replicaset", *replicaName)
	corelog.UseTimestamp(true)

	// Log command line args
	startupOptions := []interface{}{}
	flag.CommandLine.VisitAll(func(flag *flag.Flag) {
		if flag.Name != "password" {
			startupOptions = append(startupOptions, flag.Name, flag.Value.String())
		}
	})
	corelog.LogInfoMessage("starting with command line arguments", startupOptions...)

	// Wrapper for inject
	log := Logger{}

	var graph inject.Graph
	err := graph.Provide(
		&inject.Object{Value: &replicaSet},
		&inject.Object{Value: &statsClient},
		&inject.Object{Value: stateManager},
	)
	if err != nil {
		return err
	}
	if err := graph.Populate(); err != nil {
		return err
	}
	objects := graph.Objects()

	hc := &dvara.HealthChecker{
		HealthCheckInterval:        *healthCheckInterval,
		FailedHealthCheckThreshold: *failedHealthCheckThreshold,
	}

	if err := startstop.Start(objects, &log); err != nil {
		return err
	}
	defer startstop.Stop(objects, &log)

	syncChan := make(chan struct{})
	go stateManager.KeepSynchronized(syncChan)
	go hc.HealthCheck(&replicaSet, syncChan)

	ch := make(chan os.Signal, 2)
	signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
	<-ch
	signal.Stop(ch)
	return nil
}