Example #1
0
File: tzk.go Project: yylq/tcase
func test_zkcreatetemp(s string) {
	c, _, err := zk.Connect([]string{"192.168.145.224"}, time.Second) //*10)
	if err != nil {
		panic(err)
	}
	defer c.Close()
}
Example #2
0
func connect(zksStr string) (*zk.Conn, <-chan zk.Event) {
	//	zksStr := os.Getenv("ZOOKEEPER_SERVERS")
	zks := strings.Split(zksStr, ",")
	conn, connEvtChan, err := zk.Connect(zks, 5*time.Second)
	must(err)
	return conn, connEvtChan
}
// CreateTestData: Implement the zkTestCallbacks interface
// Initialize the steps
func (test *zkFollowerResignationTest) InitFunc() ([]time.Duration, error) {
	test.info.Println("InitFunc()")

	// Create the connection to Zookeeper
	zkConn, _, err := zk.Connect([]string{test.testSetup.zkURL}, test.testSetup.heartBeat)
	test.zkConn = zkConn

	if err != nil {
		test.error.Printf("frTest: Error in zk.Connect (%s): %v",
			test.testSetup.zkURL, err)
		return nil, err
	}

	// Create the election node in ZooKeeper
	myStr, err := zkConn.Create(test.testSetup.electionNode,
		[]byte("data"), 0, zk.WorldACL(zk.PermAll))
	if err != nil {
		test.error.Printf("frTest: Error creating the election node (%s): %v",
			test.testSetup.electionNode, err)
		return nil, err
	}

	fmt.Println("Got myStr from create: ", myStr)

	return []time.Duration{
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
	}, nil
}
Example #4
0
func PersistFrameworkID(
	fwid *mesos.FrameworkID,
	zkServers []string,
	zkChroot string,
	frameworkName string,
) error {
	c, _, err := zk.Connect(zkServers, RPC_TIMEOUT)
	if err != nil {
		return err
	}
	defer c.Close()
	// attempt to create the path
	_, err = c.Create(
		zkChroot,
		[]byte(""),
		0,
		zk.WorldACL(zk.PermAll),
	)
	if err != nil && err != zk.ErrNodeExists {
		return err
	}
	// attempt to write framework ID to <path> / <frameworkName>
	_, err = c.Create(zkChroot+"/"+frameworkName+"_framework_id",
		[]byte(fwid.GetValue()),
		0,
		zk.WorldACL(zk.PermAll))
	// TODO(tyler) when err is zk.ErrNodeExists, cross-check value
	if err != nil {
		return err
	}
	log.Info("Successfully persisted Framework ID to zookeeper.")

	return nil
}
// CreateTestData: Implement the zkTestCallbacks interface
// Initialize the steps
func (test *zkLeaderResignationTest) InitFunc() ([]time.Duration, error) {
	test.info.Println("InitFunc()")

	// Create the connection to Zookeeper
	zkConn, _, err := zk.Connect([]string{test.testSetup.zkURL}, test.testSetup.heartBeat)
	test.zkConn = zkConn
	test.wg = &sync.WaitGroup{}

	if err != nil {
		test.error.Printf("leaderResignationTest: Error in zk.Connect (%s): %v",
			test.testSetup.zkURL, err)
		return nil, err
	}

	// Create the election node in ZooKeeper
	_, err = test.zkConn.Create(test.testSetup.electionNode,
		[]byte("data"), 0, zk.WorldACL(zk.PermAll))
	if err != nil {
		test.error.Printf("leaderResignationTest: Error creating the election node (%s): %v",
			test.testSetup.electionNode, err)
		return nil, err
	}

	return []time.Duration{
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
		time.Second * 1,
	}, nil
}
Example #6
0
func GetPreviousFrameworkID(
	zkServers []string,
	zkChroot string,
	frameworkName string,
) (fwid string, err error) {
	request := func() (string, error) {
		c, _, err := zk.Connect(zkServers, RPC_TIMEOUT)
		if err != nil {
			return "", err
		}
		defer c.Close()
		rawData, _, err := c.Get(zkChroot + "/" + frameworkName + "_framework_id")
		return string(rawData), err
	}

	backoff := 1
	for retries := 0; retries < RPC_RETRIES; retries++ {
		fwid, err = request()
		if err == nil {
			return fwid, err
		}
		time.Sleep(time.Duration(backoff) * time.Second)
		backoff = int(math.Min(float64(backoff<<1), 8))
	}
	return "", err
}
Example #7
0
func GetPreviousReconciliationInfo(
	zkServers []string,
	zkChroot string,
	frameworkName string,
) (recon map[string]string, err error) {
	request := func() (map[string]string, error) {
		c, _, err := zk.Connect(zkServers, RPC_TIMEOUT)
		if err != nil {
			return map[string]string{}, err
		}
		defer c.Close()
		rawData, _, err := c.Get(zkChroot + "/" + frameworkName + "_reconciliation")
		if err == zk.ErrNoNode {
			return map[string]string{}, nil
		}
		if err != nil {
			return map[string]string{}, err
		}
		reconciliationInfo := map[string]string{}
		err = json.Unmarshal(rawData, &reconciliationInfo)
		return reconciliationInfo, err
	}

	backoff := 1
	for retries := 0; retries < RPC_RETRIES; retries++ {
		recon, err = request()
		if err == nil {
			return recon, err
		}
		time.Sleep(time.Duration(backoff) * time.Second)
		backoff = int(math.Min(float64(backoff<<1), 8))
	}
	return recon, err
}
Example #8
0
// DialZkTimeout dial the server, and wait up to timeout until connection
func DialZkTimeout(zkAddr string, baseTimeout time.Duration, connectTimeout time.Duration) (*ZkConn, <-chan zookeeper.Event, error) {
	servers, err := resolveZkAddr(zkAddr)
	if err != nil {
		return nil, nil, err
	}

	sem.Acquire()
	defer sem.Release()
	zconn, session, err := zookeeper.Connect(servers, baseTimeout)
	if err != nil {
		return nil, nil, err
	}

	// Wait for connection, with a timeout, skipping transition states
	timer := time.NewTimer(connectTimeout)
	for {
		select {
		case <-timer.C:
			zconn.Close()
			return nil, nil, context.DeadlineExceeded
		case event := <-session:
			switch event.State {
			case zookeeper.StateConnected:
				// success
				return &ZkConn{conn: zconn}, session, nil

			case zookeeper.StateAuthFailed:
				// fast fail this one
				zconn.Close()
				return nil, nil, fmt.Errorf("zk connect failed: StateAuthFailed")
			}
		}
	}
}
Example #9
0
func createNode() {
	const Node = "/greet"

	conn, _, _ := zk.Connect([]string{zkServer}, time.Second*5)

	create := func() error {
		var err error
		// try creating ephemeral node
		_, err = conn.Create(Node, []byte(httpPort), zk.FlagEphemeral, zk.WorldACL(zk.PermAll))
		return err
	}

	if create() != nil {
		// watch ephemeral node event.
		another, _, eventChan, _ := conn.GetW(Node)
		fmt.Println("Now listen", string(another))
	loop:
		for {
			event := <-eventChan
			if event.Type == zk.EventNodeDeleted || event.Type.String() == "Unknown" {
				// retry creating ephemeral node
				if create() != nil {
					break loop
				}
			}
		}
	}
}
Example #10
0
// DialZk dials a ZK server and waits for connection event. Returns a ZkConn
// encapsulating the zookeeper.Conn, and the zookeeper session event
// channel to monitor the connection
//
// The value for baseTimeout is used as a session timeout as well, and
// will be used to negotiate a 'good' value with the server. From
// reading the zookeeper source code, it has to be between 6 and 60
// seconds (2x and 20x the tickTime by default, with default tick time
// being 3 seconds).  min session time, max session time and ticktime
// can all be overwritten on the zookeeper server side, so these
// numbers may vary.
//
// Then this baseTimeout is used to compute other related timeouts:
// - connect timeout is 1/3 of baseTimeout
// - recv timeout is 2/3 of baseTimeout minus a ping time
// - send timeout is 1/3 of baseTimeout
// - we try to send a ping a least every baseTimeout / 3
//
// Note the baseTimeout has *nothing* to do with the time between we
// call Dial and the maximum time before we receive the event on the
// session. The library will actually try to re-connect in the background
// (after each timeout), and may *never* send an event if the TCP connections
// always fail. Use DialZkTimeout to enforce a timeout for the initial connect.
func DialZk(zkAddr string, baseTimeout time.Duration) (*ZkConn, <-chan zookeeper.Event, error) {
	servers, err := resolveZkAddr(zkAddr)
	if err != nil {
		return nil, nil, err
	}

	sem.Acquire()
	defer sem.Release()
	zconn, session, err := zookeeper.Connect(servers, baseTimeout)
	if err != nil {
		return nil, nil, err
	}

	// Wait for connection, possibly forever, skipping transition states
	for {
		event := <-session
		switch event.State {
		case zookeeper.StateConnected:
			// success
			return &ZkConn{conn: zconn}, session, nil

		case zookeeper.StateAuthFailed:
			// fast fail this one
			zconn.Close()
			return nil, nil, fmt.Errorf("zk connect failed: StateAuthFailed")
		}
	}
}
Example #11
0
func GetConnect() (conn *zk.Conn, err error) {
	conn, _, err = zk.Connect(hosts, timeOut*time.Second)
	if err != nil {
		fmt.Println(err)
	}
	return
}
Example #12
0
// a health check for zookeeper
func zkHealthCheck() error {

	start := time.Now()
	lastError := time.Now()
	minUptime := time.Second * 2
	timeout := time.Second * 30
	zookeepers := []string{"127.0.0.1:2181"}

	for {
		if conn, _, err := zk.Connect(zookeepers, time.Second*10); err == nil {
			conn.Close()
		} else {
			conn.Close()
			glog.V(1).Infof("Could not connect to zookeeper: %s", err)
			lastError = time.Now()
		}
		// make sure that service has been good for at least minUptime
		if time.Since(lastError) > minUptime {
			break
		}
		if time.Since(start) > timeout {
			return fmt.Errorf("Zookeeper did not respond.")
		}
		time.Sleep(time.Millisecond * 1000)
	}
	glog.Info("zookeeper container started, browser at http://localhost:12181/exhibitor/v1/ui/index.html")
	return nil
}
Example #13
0
func NewZookeeperClient(connectAddr string) (*Client, error) {
	c, _, err := zk.Connect([]string{connectAddr}, time.Second) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}
Example #14
0
func NewZookeeperClient(app *ApplicationContext, cluster string) (*ZookeeperClient, error) {
	zkconn, _, err := zk.Connect(app.Config.Kafka[cluster].Zookeepers, time.Duration(app.Config.Zookeeper.Timeout)*time.Second)
	if err != nil {
		return nil, err
	}

	client := &ZookeeperClient{
		app:         app,
		cluster:     cluster,
		conn:        zkconn,
		zkGroupLock: sync.RWMutex{},
		zkGroupList: make(map[string]bool),
	}

	// Check if this cluster is configured to check Zookeeper consumer offsets
	if client.app.Config.Kafka[cluster].ZKOffsets {
		// Get a group list to start with (this will start the offset checkers)
		client.refreshConsumerGroups()

		// Set a ticker to refresh the group list periodically
		client.zkRefreshTicker = time.NewTicker(time.Duration(client.app.Config.Lagcheck.ZKGroupRefresh) * time.Second)
		go func() {
			for _ = range client.zkRefreshTicker.C {
				client.refreshConsumerGroups()
			}
		}()
	}

	return client, nil
}
Example #15
0
func TestZookeeperBackend(t *testing.T) {
	addr := os.Getenv("ZOOKEEPER_ADDR")
	if addr == "" {
		t.SkipNow()
	}

	client, _, err := zk.Connect([]string{addr}, time.Second)

	if err != nil {
		t.Fatalf("err: %v", err)
	}

	randPath := fmt.Sprintf("/vault-%d", time.Now().Unix())
	acl := zk.WorldACL(zk.PermAll)
	_, err = client.Create(randPath, []byte("hi"), int32(0), acl)

	if err != nil {
		t.Fatalf("err: %v", err)
	}

	defer func() {
		client.Delete(randPath, -1)
	}()

	b, err := NewBackend("zookeeper", map[string]string{
		"address": addr + "," + addr,
		"path":    randPath,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	testBackend(t, b)
	testBackend_ListPrefix(t, b)
}
Example #16
0
func NewZookeeperClient(machines []string) (*Client, error) {
	c, _, err := zk.Connect(machines, time.Second) //*10)
	if err != nil {
		panic(err)
	}
	return &Client{c}, nil
}
Example #17
0
func InitZK() (*zk.Conn, error) {
	// connect to zookeeper, get event from chan in goroutine(log)
	Log.Debug("zk timeout: %d", Conf.ZKTimeout)
	conn, session, err := zk.Connect(Conf.ZKAddr, Conf.ZKTimeout)
	if err != nil {
		Log.Error("zk.Connect(\"%v\", %d) error(%v)", Conf.ZKAddr, Conf.ZKTimeout, err)
		return nil, err
	}
	go func() {
		for {
			event := <-session
			Log.Info("zookeeper get a event: %s", event.State.String())
		}
	}()
	// Create zk public message-lock root path
	Log.Debug("create zookeeper path: %s", Conf.ZKPIDPath)
	_, err = conn.Create(Conf.ZKPIDPath, []byte(""), 0, zk.WorldACL(zk.PermAll))
	if err != nil {
		if err == zk.ErrNodeExists {
			Log.Warn("zk.create(\"%s\") exists", Conf.ZKPIDPath)
		} else {
			Log.Error("zk.create(\"%s\") error(%v)", Conf.ZKPIDPath, err)
			return nil, err
		}
	}
	// after init nodes, then watch them
	ch := make(chan *NodeEvent, 1024)
	go handleNodeEvent(conn, Conf.ZKCometPath, ch)
	go watchRoot(conn, Conf.ZKCometPath, ch)
	return conn, nil
}
Example #18
0
func connect() *zk.Conn {
	zksStr := os.Getenv("ZOOKEEPER_SERVERS")
	zks := strings.Split(zksStr, ",")
	conn, _, err := zk.Connect(zks, time.Second)
	must(err)
	return conn
}
Example #19
0
func main() {
	numCPU := runtime.NumCPU()
	runtime.GOMAXPROCS(numCPU)

	obliteration := flag.Int("concurrency", 10, "threads and connections to use for load generation")
	host := flag.String("zk", "master.mesos:2181", "host:port for zk")
	size := flag.Int("size", 1024, "bytes per key written")
	ratio := flag.Float64("ratio", 0.2, "0 to 1 ratio of reads to writes.  0 is all writes, 1 is all reads.")
	flag.Parse()

	value := gen(*size)

	conns := []*zk.Conn{}
	for i := 0; i < *obliteration; i++ {
		cli, _, err := zk.Connect([]string{*host}, 5*time.Second)
		if err != nil {
			fmt.Printf("error connecting to zk: %v\n", err)
			os.Exit(1)
		}
		conns = append(conns, cli)
	}

	doRpc := func() {
		cli := conns[rand.Intn(len(conns))]
		bench(cli, value, *ratio)
	}

	loghisto.PrintBenchmark("benchmark1234", uint(*obliteration), doRpc)
}
Example #20
0
func (z *ZookeeperClient) Connect() {
	var err error
	z.zkConn, _, err = zk.Connect(z.config.Servers, z.config.RecvTimeout)
	if err != nil {
		fmt.Println(err)
	}
}
Example #21
0
func (zc *zkflagCheck) Connect() (zk.State, error) {
	if zc.Connection != nil {
		state := zc.Connection.State()
		switch state {
		case zk.StateUnknown, zk.StateConnectedReadOnly, zk.StateExpired, zk.StateAuthFailed, zk.StateConnecting:
			{
				//Disconnect, and let Reconnection happen
				log.Warn("ZKFlag Connection is in BAD State [", state, "] Reconnect")
				zc.Connection.Close()
			}
		case zk.StateConnected, zk.StateHasSession:
			{
				log.Debug("ZKFlag Connection established(", state, "), nothing to do.")
				return state, nil
			}
		case zk.StateDisconnected:
			{
				log.Info("ZKFlag Connection is Disconnected -> Reconnection")
			}
		}
	}
	conn, _, err := zk.Connect(zc.Hosts, 10*time.Second)
	if err != nil {
		zc.Connection = nil
		log.Warn("Unable to Connect to ZKFlag (", err, ")")
		return zk.StateDisconnected, err
	}
	zc.Connection = conn
	var zkLogger ZKDebugLogger
	zc.Connection.SetLogger(zkLogger)
	state := zc.Connection.State()
	return state, nil
}
Example #22
0
func NewStormClient(app *ApplicationContext, cluster string) (*StormClient, error) {
	// here we share the timeout w/ global zk
	zkconn, _, err := zk.Connect(app.Config.Storm[cluster].Zookeepers, time.Duration(app.Config.Zookeeper.Timeout)*time.Second)
	if err != nil {
		return nil, err
	}

	client := &StormClient{
		app:            app,
		cluster:        cluster,
		conn:           zkconn,
		stormGroupLock: sync.RWMutex{},
		stormGroupList: make(map[string]bool),
	}

	// Now get the first set of offsets and start a goroutine to continually check them
	client.refreshConsumerGroups()
	client.stormRefreshTicker = time.NewTicker(time.Duration(client.app.Config.Lagcheck.StormGroupRefresh) * time.Second)
	go func() {
		for _ = range client.stormRefreshTicker.C {
			client.refreshConsumerGroups()
		}
	}()

	return client, nil
}
Example #23
0
func (zk *ZookeeperClusterRunner) Reset() {
	client, _, err := zkClient.Connect(zk.NodeURLS(), time.Second)
	Ω(err).ShouldNot(HaveOccured(), "Failed to connect")

	zk.deleteRecursively(client, "/")
	client.Close()
}
func TestConnectToZooKeeper(t *testing.T) {
	c, err := ConnectToZooKeeper(15, time.Millisecond*500, func(url string) bool {
		conn, _, err := zk.Connect([]string{url}, time.Second)
		if err != nil {
			return false
		}
		defer conn.Close()

		// Verify that we can perform operations, and that it's
		// a clean slate (/zookeeper should be the only path)
		children, _, err := conn.Children("/")
		if err != nil {
			return false
		}
		if len(children) != 1 {
			return false
		}
		if children[0] != "zookeeper" {
			return false
		}

		return true
	})
	assert.Nil(t, err)
	defer c.KillRemove()
}
Example #25
0
// newZookeeperBackend constructs a Zookeeper backend using the given API client
// and the prefix in the KV store.
func newZookeeperBackend(conf map[string]string) (Backend, error) {
	// Get the path in Zookeeper
	path, ok := conf["path"]
	if !ok {
		path = "vault/"
	}

	// Ensure path is suffixed and prefixed (zk requires prefix /)
	if !strings.HasSuffix(path, "/") {
		path += "/"
	}
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}

	// Configure the client, default to localhost instance
	var machines string
	machines, ok = conf["address"]
	if !ok {
		machines = "localhost:2181"
	}

	// Attempt to create the ZK client
	client, _, err := zk.Connect(strings.Split(machines, ","), time.Second)
	if err != nil {
		return nil, fmt.Errorf("client setup failed: %v", err)
	}

	// Setup the backend
	c := &ZookeeperBackend{
		path:   path,
		client: client,
	}
	return c, nil
}
Example #26
0
func (c *Client) initZk() {
	zkclient, _, err := zk.Connect(c.zkHosts, time.Second*30)
	if err != nil {
		panic(err)
	}

	c.zkClient = zkclient

	res, _, _, err := c.zkClient.GetW(c.zkRoot + c.zkRootRegionPath)

	if err != nil {
		panic(err)
	}

	c.rootServer = c.decodeMeta(res)
	c.getRegionConnection(c.getServerName(c.rootServer))

	res, _, _, err = c.zkClient.GetW(c.zkRoot + "/master")

	if err != nil {
		panic(err)
	}

	c.masterServer = c.decodeMeta(res)
}
Example #27
0
func (zr *ZookeeperReporter) Connect() (zk.State, error) {
	if zr.ZKConnection != nil {
		state := zr.ZKConnection.State()
		switch state {
		case zk.StateUnknown, zk.StateConnectedReadOnly, zk.StateExpired, zk.StateAuthFailed, zk.StateConnecting:
			{
				//Disconnect, and let Reconnection happen
				log.Warn("Zookeeper Connection is in BAD State [", state, "] Reconnect")
				zr.ZKConnection.Close()
			}
		case zk.StateConnected, zk.StateHasSession:
			{
				log.Debug("Zookeeper Connection of [", zr.ServiceName, "][", zr.InstanceID, "] connected(", state, "), nothing to do.")
				return state, nil
			}
		case zk.StateDisconnected:
			{
				log.Info("Reporter Connection is Disconnected -> Reconnection")
			}
		}
	}
	conn, _, err := zk.Connect(zr.ZKHosts, 10*time.Second)
	if err != nil {
		zr.ZKConnection = nil
		log.Warn("Unable to Connect to ZooKeeper (", err, ")")
		return zk.StateDisconnected, err
	}
	zr.ZKConnection = conn
	var zkLogger ZKDebugLogger
	zr.ZKConnection.SetLogger(zkLogger)
	zr.ZKConnection = conn
	state := zr.ZKConnection.State()
	return state, nil
}
Example #28
0
func (this *ServiceBase) Connect() (err error) {
	if this.Conn != nil {
		this.Conn.Close()
	}
	this.Conn, _, err = zk.Connect(this.Hosts, this.ConnTimeout)
	return err
}
// NewZk connect zookeeper
func NewZk(zks []string) (*zk.Conn, error) {
	c, _, err := zk.Connect(zks, time.Second*1)
	if err != nil {
		return nil, err
	}
	return c, nil
}
Example #30
-1
func (mgr *MetadataManager) CreateConnection() {
	if mgr.zkConn != nil {
		// Return if the connection is good already
		if mgr.zkConn.State() == zk.StateConnected ||
			mgr.zkConn.State() == zk.StateHasSession ||
			mgr.zkConn.State() == zk.StateConnecting {
			return
		}
		// Close the connection because it probably expired
		mgr.zkConn.Close()
	}

	conn, _, err := zk.Connect(mgr.zookeepers, time.Second)
	if err != nil {
		conn.Close()
		log.Panic(err)
	}
	bns := baseNamespace{}
	ns := makeSubSpace(makeSubSpace(makeSubSpace(bns, "riak"), "frameworks"), mgr.frameworkID)
	lockPath := makeSubSpace(ns, "lock")
	zkLock := zk.NewLock(conn, lockPath.GetZKPath(), zk.WorldACL(zk.PermAll))

	mgr.zkConn = conn
	mgr.namespace = ns
	mgr.zkLock = *zkLock
}