Esempio n. 1
0
func NewTestServer(t *testing.T, cells []string) *TestServer {
	zconn := fakezk.NewConn()

	// create the toplevel zk paths
	if _, err := zk.CreateRecursive(zconn, "/zk/global/vt", "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
		t.Fatalf("cannot init ZooKeeper: %v", err)
	}
	for _, cell := range cells {
		if _, err := zk.CreateRecursive(zconn, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init ZooKeeper: %v", err)
		}
	}
	return &TestServer{Server: NewServer(zconn), localCells: cells}
}
Esempio n. 2
0
func (zkts *Server) CreateKeyspace(keyspace string, value *topo.Keyspace) error {
	keyspacePath := path.Join(globalKeyspacesPath, keyspace)
	pathList := []string{
		keyspacePath,
		path.Join(keyspacePath, "action"),
		path.Join(keyspacePath, "actionlog"),
		path.Join(keyspacePath, "shards"),
	}

	alreadyExists := false
	for i, zkPath := range pathList {
		c := ""
		if i == 0 {
			c = jscfg.ToJson(value)
		}
		_, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
		if err != nil {
			if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
				alreadyExists = true
			} else {
				return fmt.Errorf("error creating keyspace: %v %v", zkPath, err)
			}
		}
	}
	if alreadyExists {
		return topo.ErrNodeExists
	}

	event.Dispatch(&events.KeyspaceChange{
		KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, value, -1),
		Status:       "created",
	})
	return nil
}
Esempio n. 3
0
// NewConnFromFile returns a fake zk.Conn implementation, that is seeded
// with the json data extracted from the input file.
func NewConnFromFile(filename string) zk.Conn {
	result := &zconn{
		root: &stat{
			name:     "/",
			children: make(map[string]*stat),
		},
		existWatches: make(map[string][]chan zookeeper.Event)}
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(fmt.Errorf("NewConnFromFile failed to read file %v: %v", filename, err))
	}
	values := make(map[string]interface{})
	if err := json.Unmarshal(data, &values); err != nil {
		panic(fmt.Errorf("NewConnFromFile failed to json.Unmarshal file %v: %v", filename, err))
	}
	for k, v := range values {
		jv, err := json.Marshal(v)
		if err != nil {
			panic(fmt.Errorf("NewConnFromFile failed to json.Marshal value %v: %v", k, err))
		}

		// CreateRecursive will work for a leaf node where the parent
		// doesn't exist, but not for a node in the middle of a tree
		// that already exists. So have to use 'Set' as a backup.
		if _, err := zk.CreateRecursive(result, k, string(jv), 0, nil); err != nil {
			if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
				_, err = result.Set(k, string(jv), -1)
			}
			if err != nil {
				panic(fmt.Errorf("NewConnFromFile failed to zk.CreateRecursive value %v: %v", k, err))
			}
		}
	}
	return result
}
Esempio n. 4
0
func (zkts *Server) UpdateShardReplicationFields(cell, keyspace, shard string, update func(*topo.ShardReplication) error) error {
	// create the parent directory to be sure it's here
	zkDir := path.Join("/zk", cell, "vt", "replication", keyspace)
	if _, err := zk.CreateRecursive(zkts.zconn, zkDir, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
		return err
	}

	// now update the data
	zkPath := shardReplicationPath(cell, keyspace, shard)
	f := func(oldValue string, oldStat zk.Stat) (string, error) {
		sr := &topo.ShardReplication{}
		if oldValue != "" {
			if err := json.Unmarshal([]byte(oldValue), sr); err != nil {
				return "", err
			}
		}

		if err := update(sr); err != nil {
			return "", err
		}
		return jscfg.ToJson(sr), nil
	}
	err := zkts.zconn.RetryChange(zkPath, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNONODE) {
			err = topo.ErrNoNode
		}
		return err
	}
	return nil
}
Esempio n. 5
0
func (zkts *Server) UpdateSrvKeyspace(cell, keyspace string, srvKeyspace *topo.SrvKeyspace) error {
	path := zkPathForVtKeyspace(cell, keyspace)
	data := jscfg.ToJson(srvKeyspace)
	_, err := zkts.zconn.Set(path, data, -1)
	if zookeeper.IsError(err, zookeeper.ZNONODE) {
		_, err = zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	}
	return err
}
Esempio n. 6
0
func newFakeTeeServer(t *testing.T) topo.Server {
	cells := []string{"test", "global"} // global has to be last

	zconn1 := fakezk.NewConn()
	zconn2 := fakezk.NewConn()

	for _, cell := range cells {
		if _, err := zk.CreateRecursive(zconn1, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init ZooKeeper: %v", err)
		}
		if _, err := zk.CreateRecursive(zconn2, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init ZooKeeper: %v", err)
		}
	}
	s1 := fakeServer{Server: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]}
	s2 := fakeServer{Server: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]}

	return NewTee(s1, s2, false)
}
Esempio n. 7
0
func setPathData(filePath, data string) error {
	if isZkFile(filePath) {
		_, err := zconn.Set(filePath, data, -1)
		if err != nil && zookeeper.IsError(err, zookeeper.ZNONODE) {
			_, err = zk.CreateRecursive(zconn, filePath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
		}
		return err
	} else {
		return ioutil.WriteFile(filePath, []byte(data), 0666)
	}
}
Esempio n. 8
0
func (zkts *Server) unlockForAction(lockPath, results string) error {
	// Write the data to the actionlog
	actionLogPath := strings.Replace(lockPath, "/action/", "/actionlog/", 1)
	if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath, results, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
		log.Warningf("Cannot create actionlog path %v (check the permissions with 'zk stat'), will keep the lock, use 'zk rm' to clear the lock", actionLogPath)
		return err
	}

	// and delete the action
	return zk.DeleteRecursive(zkts.zconn, lockPath, -1)
}
Esempio n. 9
0
func (zkts *Server) UpdateEndPoints(cell, keyspace, shard string, tabletType topo.TabletType, addrs *topo.EndPoints) error {
	path := zkPathForVtName(cell, keyspace, shard, tabletType)
	data := jscfg.ToJson(addrs)
	_, err := zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			// Node already exists - just stomp away. Multiple writers shouldn't be here.
			// We use RetryChange here because it won't update the node unnecessarily.
			f := func(oldValue string, oldStat zk.Stat) (string, error) {
				return data, nil
			}
			err = zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f)
		}
	}
	return err
}
Esempio n. 10
0
func cmdTouch(subFlags *flag.FlagSet, args []string) {
	var (
		createParents = subFlags.Bool("p", false, "create parents")
		touchOnly     = subFlags.Bool("c", false, "touch only - don't create")
	)

	subFlags.Parse(args)
	if subFlags.NArg() != 1 {
		log.Fatal("touch: need to specify exactly one path")
	}

	zkPath := fixZkPath(subFlags.Arg(0))
	if !isZkFile(zkPath) {
		log.Fatalf("touch: not a /zk file %v", zkPath)
	}

	var (
		version = -1
		create  = false
	)

	data, stat, err := zconn.Get(zkPath)
	switch {
	case err == nil:
		version = stat.Version()
	case zookeeper.IsError(err, zookeeper.ZNONODE):
		create = true
	default:
		log.Fatalf("touch: cannot access %v: %v", zkPath, err)
	}

	switch {
	case !create:
		_, err = zconn.Set(zkPath, data, version)
	case *touchOnly:
		log.Fatalf("touch: no such path %v", zkPath)
	case *createParents:
		_, err = zk.CreateRecursive(zconn, zkPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	default:
		_, err = zconn.Create(zkPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	}

	if err != nil {
		log.Fatalf("touch: cannot modify %v: %v", zkPath, err)
	}
}
Esempio n. 11
0
func (zkts *Server) LockSrvShardForAction(cell, keyspace, shard, contents string, timeout time.Duration, interrupted chan struct{}) (string, error) {
	// Action paths end in a trailing slash to that when we create
	// sequential nodes, they are created as children, not siblings.
	actionDir := path.Join(zkPathForVtShard(cell, keyspace, shard), "action")

	// if we can't create the lock file because the directory doesn't exist,
	// create it
	p, err := zkts.lockForAction(actionDir+"/", contents, timeout, interrupted)
	if err != nil && zookeeper.IsError(err, zookeeper.ZNONODE) {
		_, err = zk.CreateRecursive(zkts.zconn, actionDir, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
		if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			return "", err
		}
		p, err = zkts.lockForAction(actionDir+"/", contents, timeout, interrupted)
	}
	return p, err
}
Esempio n. 12
0
func (zkts *Server) CreateTablet(tablet *topo.Tablet) error {
	zkTabletPath := TabletPathForAlias(tablet.Alias)

	// Create /zk/<cell>/vt/tablets/<uid>
	_, err := zk.CreateRecursive(zkts.zconn, zkTabletPath, tablet.Json(), 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
	if err != nil {
		if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			err = topo.ErrNodeExists
		}
		return err
	}

	event.Dispatch(&events.TabletChange{
		Tablet: *tablet,
		Status: "created",
	})
	return nil
}
Esempio n. 13
0
func cmdUnzip(subFlags *flag.FlagSet, args []string) {
	subFlags.Parse(args)
	if subFlags.NArg() != 2 {
		log.Fatalf("zip: need to specify source and destination paths")
	}

	srcPath, dstPath := subFlags.Arg(0), subFlags.Arg(1)

	if !strings.HasSuffix(srcPath, ".zip") {
		log.Fatalf("zip: need to specify src .zip path: %v", srcPath)
	}

	zipReader, err := zip.OpenReader(srcPath)
	if err != nil {
		log.Fatalf("zip: error %v", err)
	}
	defer zipReader.Close()

	for _, zf := range zipReader.File {
		rc, err := zf.Open()
		if err != nil {
			log.Fatalf("unzip: error %v", err)
		}
		data, err := ioutil.ReadAll(rc)
		if err != nil {
			log.Fatalf("unzip: failed reading archive: %v", err)
		}
		zkPath := zf.Name
		if dstPath != "/" {
			zkPath = path.Join(dstPath, zkPath)
		}
		_, err = zk.CreateRecursive(zconn, zkPath, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL))
		if err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) {
			log.Fatalf("unzip: zk create failed: %v", err)
		}
		_, err = zconn.Set(zkPath, string(data), -1)
		if err != nil {
			log.Fatalf("unzip: zk set failed: %v", err)
		}
		rc.Close()
	}
}
Esempio n. 14
0
func createSetup(t *testing.T) (topo.Server, topo.Server) {
	fromConn := fakezk.NewConn()
	fromTS := zktopo.NewServer(fromConn)

	toConn := fakezk.NewConn()
	toTS := zktopo.NewServer(toConn)

	for _, zkPath := range []string{"/zk/test_cell/vt", "/zk/global/vt"} {
		if _, err := zk.CreateRecursive(fromConn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil {
			t.Fatalf("cannot init fromTS: %v", err)
		}
	}

	// create a keyspace and a couple tablets
	if err := fromTS.CreateKeyspace("test_keyspace", &topo.Keyspace{}); err != nil {
		t.Fatalf("cannot create keyspace: %v", err)
	}
	if err := fromTS.CreateShard("test_keyspace", "0", &topo.Shard{Cells: []string{"test_cell"}}); err != nil {
		t.Fatalf("cannot create shard: %v", err)
	}
	if err := topo.CreateTablet(fromTS, &topo.Tablet{
		Alias: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  123,
		},
		Hostname: "masterhost",
		Parent:   topo.TabletAlias{},
		IPAddr:   "1.2.3.4",
		Portmap: map[string]int{
			"vt":    8101,
			"vts":   8102,
			"mysql": 3306,
		},
		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topo.TYPE_MASTER,
		State:          topo.STATE_READ_WRITE,
		DbNameOverride: "",
		KeyRange:       key.KeyRange{},
	}); err != nil {
		t.Fatalf("cannot create master tablet: %v", err)
	}
	if err := topo.CreateTablet(fromTS, &topo.Tablet{
		Alias: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  234,
		},
		IPAddr: "2.3.4.5",
		Portmap: map[string]int{
			"vt":    8101,
			"vts":   8102,
			"mysql": 3306,
		},
		Hostname: "slavehost",

		Parent: topo.TabletAlias{
			Cell: "test_cell",
			Uid:  123,
		},
		Keyspace:       "test_keyspace",
		Shard:          "0",
		Type:           topo.TYPE_REPLICA,
		State:          topo.STATE_READ_ONLY,
		DbNameOverride: "",
		KeyRange:       key.KeyRange{},
	}); err != nil {
		t.Fatalf("cannot create slave tablet: %v", err)
	}

	os.Setenv("ZK_CLIENT_CONFIG", testfiles.Locate("topo_helpers_test_zk_client.json"))
	cells, err := fromTS.GetKnownCells()
	if err != nil {
		t.Fatalf("fromTS.GetKnownCells: %v", err)
	}
	log.Infof("Cells: %v", cells)

	return fromTS, toTS
}