// TestPruneActionLogs is a ZK specific unit test func TestPruneActionLogs(t *testing.T) { ctx := context.Background() ts := NewTestServer(t, []string{"test"}) defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &topo.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } actionLogPath := path.Join(globalKeyspacesPath, "test_keyspace", "actionlog") zkts := ts.Server.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath+"/0", "first", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(stale): %v", err) } if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath+"/1", "second", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(fresh): %v", err) } if count, err := zkts.PruneActionLogs(actionLogPath, 1); err != nil || count != 1 { t.Fatalf("PruneActionLogs: %v %v", err, count) } actionLogs, _, err := zkts.zconn.Children(actionLogPath) if err != nil || len(actionLogs) != 1 || actionLogs[0] != "1" { t.Errorf("PruneActionLogs kept the wrong things: %v %v", err, actionLogs) } }
// TestPurgeActions is a ZK specific unit test func TestPurgeActions(t *testing.T) { ctx := context.Background() ts := NewTestServer(t, []string{"test"}) defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &topo.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } actionPath := path.Join(globalKeyspacesPath, "test_keyspace", "action") zkts := ts.Server.(*Server) if _, err := zk.CreateRecursive(zkts.zconn, actionPath+"/topurge", "purgeme", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(topurge): %v", err) } if _, err := zk.CreateRecursive(zkts.zconn, actionPath+"/tokeep", "keepme", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("CreateRecursive(tokeep): %v", err) } if err := zkts.PurgeActions(actionPath, func(data string) bool { return data == "purgeme" }); err != nil { t.Fatalf("PurgeActions(tokeep): %v", err) } actions, _, err := zkts.zconn.Children(actionPath) if err != nil || len(actions) != 1 || actions[0] != "tokeep" { t.Errorf("PurgeActions kept the wrong things: %v %v", err, actions) } }
// TestPurgeActions is a ZK specific unit test func TestPurgeActions(t *testing.T) { ctx := context.Background() ts := newTestServer(t, []string{"test"}) defer ts.Close() if err := ts.CreateKeyspace(ctx, "test_keyspace", &topodatapb.Keyspace{}); err != nil { t.Fatalf("CreateKeyspace: %v", err) } actionPath := path.Join(zktopo.GlobalKeyspacesPath, "test_keyspace", "action") zkts := ts.(*TestServer).Impl.(*zktopo.Server) if _, err := zk.CreateRecursive(zkts.GetZConn(), actionPath+"/topurge", []byte("purgeme"), 0, zookeeper.WorldACL(zookeeper.PermAll)); err != nil { t.Fatalf("CreateRecursive(topurge): %v", err) } if _, err := zk.CreateRecursive(zkts.GetZConn(), actionPath+"/tokeep", []byte("keepme"), 0, zookeeper.WorldACL(zookeeper.PermAll)); err != nil { t.Fatalf("CreateRecursive(tokeep): %v", err) } if err := zkts.PurgeActions(actionPath, func(data []byte) bool { return string(data) == "purgeme" }); err != nil { t.Fatalf("PurgeActions(tokeep): %v", err) } actions, _, err := zkts.GetZConn().Children(actionPath) if err != nil || len(actions) != 1 || actions[0] != "tokeep" { t.Errorf("PurgeActions kept the wrong things: %v %v", err, actions) } }
// newTestServer returns a new TestServer (with the required paths created) func newTestServer(t *testing.T, cells []string) topo.Impl { zconn := fakezk.NewConn() // create the toplevel zk paths if _, err := zk.CreateRecursive(zconn, "/zk/global/vt", nil, 0, zookeeper.WorldACL(zookeeper.PermAll)); err != nil { t.Fatalf("cannot init ZooKeeper: %v", err) } for _, cell := range cells { if _, err := zk.CreateRecursive(zconn, fmt.Sprintf("/zk/%v/vt", cell), nil, 0, zookeeper.WorldACL(zookeeper.PermAll)); err != nil { t.Fatalf("cannot init ZooKeeper: %v", err) } } return &TestServer{Impl: zktopo.NewServer(zconn), localCells: cells} }
func (zkts *Server) CreateKeyspace(keyspace string) error { keyspacePath := path.Join(globalKeyspacesPath, keyspace) pathList := []string{ keyspacePath, path.Join(keyspacePath, "action"), path.Join(keyspacePath, "actionlog"), path.Join(keyspacePath, "shards"), } alreadyExists := false for _, zkPath := range pathList { _, err := zk.CreateRecursive(zkts.zconn, zkPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating keyspace: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } return nil }
func (zkts *Server) CreateTablet(tablet *topo.Tablet) error { zkTabletPath := TabletPathForAlias(tablet.Alias()) // Create /zk/<cell>/vt/tablets/<uid> _, err := zk.CreateRecursive(zkts.zconn, zkTabletPath, tablet.Json(), 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { err = topo.ErrNodeExists } return err } // Create /zk/<cell>/vt/tablets/<uid>/action tap := path.Join(zkTabletPath, "action") _, err = zkts.zconn.Create(tap, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { return err } // Create /zk/<cell>/vt/tablets/<uid>/actionlog talp := path.Join(zkTabletPath, "actionlog") _, err = zkts.zconn.Create(talp, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { return err } return nil }
// CreateShard is part of the topo.Server interface func (zkts *Server) CreateShard(ctx context.Context, keyspace, shard string, value *pb.Shard) error { shardPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard) pathList := []string{ shardPath, path.Join(shardPath, "action"), path.Join(shardPath, "actionlog"), } alreadyExists := false for i, zkPath := range pathList { c := "" if i == 0 { c = jscfg.ToJSON(value) } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating shard: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } event.Dispatch(&events.ShardChange{ ShardInfo: *topo.NewShardInfo(keyspace, shard, value, -1), Status: "created", }) return nil }
func (zkts *Server) CreateKeyspace(keyspace string, value *topo.Keyspace) error { keyspacePath := path.Join(globalKeyspacesPath, keyspace) pathList := []string{ keyspacePath, path.Join(keyspacePath, "action"), path.Join(keyspacePath, "actionlog"), path.Join(keyspacePath, "shards"), } alreadyExists := false for i, zkPath := range pathList { c := "" if i == 0 { c = jscfg.ToJson(value) } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating keyspace: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } event.Dispatch(&events.KeyspaceChange{ KeyspaceInfo: *topo.NewKeyspaceInfo(keyspace, value), Status: "created", }) return nil }
// UpdateShardReplicationFields is part of the topo.Server interface func (zkts *Server) UpdateShardReplicationFields(ctx context.Context, cell, keyspace, shard string, update func(*topodatapb.ShardReplication) error) error { // create the parent directory to be sure it's here zkDir := path.Join("/zk", cell, "vt", "replication", keyspace) if _, err := zk.CreateRecursive(zkts.zconn, zkDir, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil && !zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { return convertError(err) } // now update the data zkPath := shardReplicationPath(cell, keyspace, shard) f := func(oldValue string, oldStat zk.Stat) (string, error) { sr := &topodatapb.ShardReplication{} if oldValue != "" { if err := json.Unmarshal([]byte(oldValue), sr); err != nil { return "", err } } if err := update(sr); err != nil { return "", err } data, err := json.MarshalIndent(sr, "", " ") if err != nil { return "", err } return string(data), nil } err := zkts.zconn.RetryChange(zkPath, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f) if err != nil { return convertError(err) } return nil }
// CreateShard is part of the topo.Server interface func (zkts *Server) CreateShard(ctx context.Context, keyspace, shard string, value *topodatapb.Shard) error { shardPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard) pathList := []string{ shardPath, path.Join(shardPath, "action"), path.Join(shardPath, "actionlog"), } data, err := json.MarshalIndent(value, "", " ") if err != nil { return err } alreadyExists := false for i, zkPath := range pathList { c := "" if i == 0 { c = string(data) } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating shard: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } return nil }
// CreateKeyspace is part of the topo.Server interface func (zkts *Server) CreateKeyspace(ctx context.Context, keyspace string, value *topodatapb.Keyspace) error { keyspacePath := path.Join(GlobalKeyspacesPath, keyspace) pathList := []string{ keyspacePath, path.Join(keyspacePath, "action"), path.Join(keyspacePath, "actionlog"), path.Join(keyspacePath, "shards"), } data, err := json.MarshalIndent(value, "", " ") if err != nil { return err } alreadyExists := false for i, zkPath := range pathList { var c []byte if i == 0 { c = data } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PermAll)) switch err { case nil: // nothing here case zookeeper.ErrNodeExists: alreadyExists = true default: return convertError(err) } } if alreadyExists { return topo.ErrNodeExists } return nil }
// NewConnFromFile returns a fake zk.Conn implementation, that is seeded // with the json data extracted from the input file. func NewConnFromFile(filename string) zk.Conn { result := &zconn{ root: &node{ _stat: _stat{name: "/"}, children: make(map[string]*node), }, existWatches: make(map[string][]chan zookeeper.Event)} data, err := ioutil.ReadFile(filename) if err != nil { panic(fmt.Errorf("NewConnFromFile failed to read file %v: %v", filename, err)) } values := make(map[string]interface{}) if err := json.Unmarshal(data, &values); err != nil { panic(fmt.Errorf("NewConnFromFile failed to json.Unmarshal file %v: %v", filename, err)) } for k, v := range values { jv, err := json.Marshal(v) if err != nil { panic(fmt.Errorf("NewConnFromFile failed to json.Marshal value %v: %v", k, err)) } // CreateRecursive will work for a leaf node where the parent // doesn't exist, but not for a node in the middle of a tree // that already exists. So have to use 'Set' as a backup. if _, err := zk.CreateRecursive(result, k, string(jv), 0, nil); err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { _, err = result.Set(k, string(jv), -1) } if err != nil { panic(fmt.Errorf("NewConnFromFile failed to zk.CreateRecursive value %v: %v", k, err)) } } } return result }
// UpdateEndPoints is part of the topo.Server interface func (zkts *Server) UpdateEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType topodatapb.TabletType, addrs *topodatapb.EndPoints, existingVersion int64) error { path := zkPathForVtName(cell, keyspace, shard, tabletType) data, err := json.MarshalIndent(addrs, "", " ") if err != nil { return err } if existingVersion == -1 { // Update or create unconditionally. _, err := zk.CreateRecursive(zkts.zconn, path, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { // Node already exists - just stomp away. Multiple writers shouldn't be here. // We use RetryChange here because it won't update the node unnecessarily. f := func(oldValue string, oldStat zk.Stat) (string, error) { return string(data), nil } err = zkts.zconn.RetryChange(path, 0, zookeeper.WorldACL(zookeeper.PERM_ALL), f) } } return err } // Compare And Set if _, err = zkts.zconn.Set(path, string(data), int(existingVersion)); err != nil { if zookeeper.IsError(err, zookeeper.ZBADVERSION) { err = topo.ErrBadVersion } else if zookeeper.IsError(err, zookeeper.ZNONODE) { err = topo.ErrNoNode } } return err }
func (zkts *Server) CreateShard(keyspace, shard string) error { shardPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard) pathList := []string{ shardPath, path.Join(shardPath, "action"), path.Join(shardPath, "actionlog"), } alreadyExists := false for i, zkPath := range pathList { c := "" if i == 0 { s := topo.Shard{} c = s.Json() } _, err := zk.CreateRecursive(zkts.zconn, zkPath, c, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { alreadyExists = true } else { return fmt.Errorf("error creating shard: %v %v", zkPath, err) } } } if alreadyExists { return topo.ErrNodeExists } return nil }
func (zkts *Server) UpdateSrvKeyspace(cell, keyspace string, srvKeyspace *topo.SrvKeyspace) error { path := zkPathForVtKeyspace(cell, keyspace) data := jscfg.ToJson(srvKeyspace) _, err := zkts.zconn.Set(path, data, -1) if zookeeper.IsError(err, zookeeper.ZNONODE) { _, err = zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) } return err }
func newFakeTeeServer(t *testing.T) topo.Server { cells := []string{"test", "global"} // global has to be last zconn1 := fakezk.NewConn() zconn2 := fakezk.NewConn() for _, cell := range cells { if _, err := zk.CreateRecursive(zconn1, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("cannot init ZooKeeper: %v", err) } if _, err := zk.CreateRecursive(zconn2, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("cannot init ZooKeeper: %v", err) } } s1 := fakeServer{Server: zktopo.NewServer(zconn1), localCells: cells[:len(cells)-1]} s2 := fakeServer{Server: zktopo.NewServer(zconn2), localCells: cells[:len(cells)-1]} return NewTee(s1, s2, false) }
// StoreTabletActionResponse stores the data both in action and actionlog func (zkts *Server) StoreTabletActionResponse(actionPath, data string) error { _, err := zkts.zconn.Set(actionPath, data, -1) if err != nil { return err } actionLogPath := strings.Replace(actionPath, "/action/", "/actionlog/", 1) _, err = zk.CreateRecursive(zkts.zconn, actionLogPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) return err }
func setPathData(filePath, data string) error { if isZkFile(filePath) { _, err := zconn.Set(filePath, data, -1) if err != nil && zookeeper.IsError(err, zookeeper.ZNONODE) { _, err = zk.CreateRecursive(zconn, filePath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) } return err } return ioutil.WriteFile(filePath, []byte(data), 0666) }
func (zkts *Server) unlockForAction(lockPath, results string) error { // Write the data to the actionlog actionLogPath := strings.Replace(lockPath, "/action/", "/actionlog/", 1) if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath, results, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { return err } // and delete the action return zk.DeleteRecursive(zkts.zconn, lockPath, -1) }
func setPathData(filePath string, data []byte) error { if isZkFile(filePath) { _, err := zconn.Set(filePath, data, -1) if err == zookeeper.ErrNoNode { _, err = zk.CreateRecursive(zconn, filePath, data, 0, zookeeper.WorldACL(zookeeper.PermAll)) } return err } return ioutil.WriteFile(filePath, []byte(data), 0666) }
// UpdateShardReplicationFields is part of the topo.Server interface func (zkts *Server) UpdateShardReplicationFields(ctx context.Context, cell, keyspace, shard string, update func(*topodatapb.ShardReplication) error) error { // create the parent directory to be sure it's here zkDir := path.Join("/zk", cell, "vt", "replication", keyspace) if _, err := zk.CreateRecursive(zkts.zconn, zkDir, nil, 0, zookeeper.WorldACL(zookeeper.PermAll)); err != nil && err != zookeeper.ErrNodeExists { return convertError(err) } // now update the data zkPath := shardReplicationPath(cell, keyspace, shard) for { data, stat, err := zkts.zconn.Get(zkPath) var version int32 = -1 sr := &topodatapb.ShardReplication{} switch err { case zookeeper.ErrNoNode: // empty node, version is 0 case nil: version = stat.Version if len(data) > 0 { if err = json.Unmarshal(data, sr); err != nil { return fmt.Errorf("bad ShardReplication data %v", err) } } default: return convertError(err) } err = update(sr) switch err { case topo.ErrNoUpdateNeeded: return nil case nil: // keep going default: return err } // marshall and save d, err := json.MarshalIndent(sr, "", " ") if err != nil { return err } if version == -1 { _, err = zkts.zconn.Create(zkPath, d, 0, zookeeper.WorldACL(zookeeper.PermAll)) if err != zookeeper.ErrNodeExists { return convertError(err) } } else { _, err = zkts.zconn.Set(zkPath, d, version) if err != zookeeper.ErrBadVersion { return convertError(err) } } } }
func newFakeServer(t *testing.T) topo.Server { zconn := fakezk.NewConn() cells := []string{"test", "global"} // global has to be last for _, cell := range cells { if _, err := zk.CreateRecursive(zconn, fmt.Sprintf("/zk/%v/vt", cell), "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { t.Fatalf("cannot init ZooKeeper: %v", err) } } return fakeServer{Server: NewServer(zconn), localCells: cells[:len(cells)-1]} }
func (zkts *Server) unlockForAction(lockPath, results string) error { // Write the data to the actionlog actionLogPath := strings.Replace(lockPath, "/action/", "/actionlog/", 1) if _, err := zk.CreateRecursive(zkts.zconn, actionLogPath, results, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)); err != nil { log.Warningf("Cannot create actionlog path %v (check the permissions with 'zk stat'), will keep the lock, use 'zk rm' to clear the lock", actionLogPath) return err } // and delete the action return zk.DeleteRecursive(zkts.zconn, lockPath, -1) }
func (zkts *Server) CreateReplicationPath(keyspace, shard, repPath string) error { replicationPath := path.Join(globalKeyspacesPath, keyspace, "shards", shard, repPath) _, err := zk.CreateRecursive(zkts.zconn, replicationPath, "", 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { err = topo.ErrNodeExists } return err } return nil }
// CreateEndPoints is part of the topo.Server interface func (zkts *Server) CreateEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType topo.TabletType, addrs *pb.EndPoints) error { path := zkPathForVtName(cell, keyspace, shard, tabletType) data := jscfg.ToJSON(addrs) // Create only if it doesn't exist. _, err := zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { err = topo.ErrNodeExists } return err }
// UpdateSrvKeyspace is part of the topo.Server interface func (zkts *Server) UpdateSrvKeyspace(ctx context.Context, cell, keyspace string, srvKeyspace *topodatapb.SrvKeyspace) error { path := zkPathForSrvKeyspace(cell, keyspace) data, err := json.MarshalIndent(srvKeyspace, "", " ") if err != nil { return err } _, err = zkts.zconn.Set(path, data, -1) if err == zookeeper.ErrNoNode { _, err = zk.CreateRecursive(zkts.zconn, path, data, 0, zookeeper.WorldACL(zookeeper.PermAll)) } return convertError(err) }
// UpdateSrvKeyspace is part of the topo.Server interface func (zkts *Server) UpdateSrvKeyspace(ctx context.Context, cell, keyspace string, srvKeyspace *topodatapb.SrvKeyspace) error { path := zkPathForVtKeyspace(cell, keyspace) data, err := json.MarshalIndent(srvKeyspace, "", " ") if err != nil { return err } _, err = zkts.zconn.Set(path, string(data), -1) if zookeeper.IsError(err, zookeeper.ZNONODE) { _, err = zk.CreateRecursive(zkts.zconn, path, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) } return err }
// UpdateSrvVSchema is part of the topo.Server interface func (zkts *Server) UpdateSrvVSchema(ctx context.Context, cell string, srvVSchema *vschemapb.SrvVSchema) error { path := zkPathForSrvVSchema(cell) data, err := json.MarshalIndent(srvVSchema, "", " ") if err != nil { return err } _, err = zkts.zconn.Set(path, string(data), -1) if err == zookeeper.ErrNoNode { _, err = zk.CreateRecursive(zkts.zconn, path, string(data), 0, zookeeper.WorldACL(zookeeper.PermAll)) } return convertError(err) }
func (zkts *Server) CreateShardReplication(cell, keyspace, shard string, sr *topo.ShardReplication) error { data := jscfg.ToJson(sr) zkPath := shardReplicationPath(cell, keyspace, shard) _, err := zk.CreateRecursive(zkts.zconn, zkPath, data, 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if err != nil { if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { err = topo.ErrNodeExists } return err } return nil }
// CreateEndPoints is part of the topo.Server interface func (zkts *Server) CreateEndPoints(ctx context.Context, cell, keyspace, shard string, tabletType topodatapb.TabletType, addrs *topodatapb.EndPoints) error { path := zkPathForVtName(cell, keyspace, shard, tabletType) data, err := json.MarshalIndent(addrs, "", " ") if err != nil { return err } // Create only if it doesn't exist. _, err = zk.CreateRecursive(zkts.zconn, path, string(data), 0, zookeeper.WorldACL(zookeeper.PERM_ALL)) if zookeeper.IsError(err, zookeeper.ZNODEEXISTS) { err = topo.ErrNodeExists } return err }