// Ensure that the cluster configuration can be reloaded. func TestClusterConfigReload(t *testing.T) { procAttr := &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}} argGroup, etcds, err := CreateCluster(3, procAttr, false) assert.NoError(t, err) defer DestroyCluster(etcds) resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":3, "removeDelay":60}`)) assert.Equal(t, resp.StatusCode, 200) time.Sleep(1 * time.Second) resp, _ = tests.Get("http://localhost:7002/v2/admin/config") body := tests.ReadBodyJSON(resp) assert.Equal(t, resp.StatusCode, 200) assert.Equal(t, body["activeSize"], 3) assert.Equal(t, body["removeDelay"], 60) // kill all DestroyCluster(etcds) for i := 0; i < 3; i++ { etcds[i], err = os.StartProcess(EtcdBinPath, argGroup[i], procAttr) } time.Sleep(1 * time.Second) resp, _ = tests.Get("http://localhost:7002/v2/admin/config") body = tests.ReadBodyJSON(resp) assert.Equal(t, resp.StatusCode, 200) assert.Equal(t, body["activeSize"], 3) assert.Equal(t, body["removeDelay"], 60) }
// Ensure that we can start a v2 cluster from the logs of a v1 cluster. func TestV1ClusterMigration(t *testing.T) { path, _ := ioutil.TempDir("", "etcd-") os.RemoveAll(path) defer os.RemoveAll(path) nodes := []string{"node0", "node2"} for i, node := range nodes { nodepath := filepath.Join(path, node) fixturepath, _ := filepath.Abs(filepath.Join("../fixtures/v1.cluster/", node)) fmt.Println("FIXPATH =", fixturepath) fmt.Println("NODEPATH =", nodepath) os.MkdirAll(filepath.Dir(nodepath), 0777) // Copy over fixture files. c := exec.Command("cp", "-rf", fixturepath, nodepath) if out, err := c.CombinedOutput(); err != nil { fmt.Println(">>>>>>\n", string(out), "<<<<<<") panic("Fixture initialization error:" + err.Error()) } procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} args := []string{"etcd", fmt.Sprintf("-data-dir=%s", nodepath)} args = append(args, "-addr", fmt.Sprintf("127.0.0.1:%d", 4001+i)) args = append(args, "-peer-addr", fmt.Sprintf("127.0.0.1:%d", 7001+i)) args = append(args, "-name", node) process, err := os.StartProcess(EtcdBinPath, args, procAttr) if err != nil { t.Fatal("start process failed:" + err.Error()) return } defer process.Kill() time.Sleep(time.Second) } // Ensure deleted message is removed. resp, err := tests.Get("http://localhost:4001/v2/keys/message") body := tests.ReadBody(resp) assert.Nil(t, err, "") assert.Equal(t, resp.StatusCode, http.StatusNotFound) assert.Equal(t, string(body), `{"errorCode":100,"message":"Key not found","cause":"/message","index":11}`+"\n") // Ensure TTL'd message is removed. resp, err = tests.Get("http://localhost:4001/v2/keys/foo") body = tests.ReadBody(resp) assert.Nil(t, err, "") assert.Equal(t, resp.StatusCode, 200, "") assert.Equal(t, string(body), `{"action":"get","node":{"key":"/foo","value":"one","modifiedIndex":9,"createdIndex":9}}`) }
// Ensures that a directory of values can be recursively retrieved for a given key. // // $ curl -X PUT localhost:4001/v2/keys/foo/x -d value=XXX // $ curl -X PUT localhost:4001/v2/keys/foo/y/z -d value=YYY // $ curl localhost:4001/v2/keys/foo -d recursive=true // func TestV2GetKeyRecursively(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") v.Set("ttl", "10") resp, _ := tests.PutForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/x"), v) tests.ReadBody(resp) v.Set("value", "YYY") resp, _ = tests.PutForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/y/z"), v) tests.ReadBody(resp) resp, _ = tests.Get(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo?recursive=true")) body := tests.ReadBodyJSON(resp) assert.Equal(t, body["action"], "get", "") assert.Equal(t, body["key"], "/foo", "") assert.Equal(t, body["dir"], true, "") assert.Equal(t, body["modifiedIndex"], 1, "") assert.Equal(t, len(body["kvs"].([]interface{})), 2, "") kv0 := body["kvs"].([]interface{})[0].(map[string]interface{}) assert.Equal(t, kv0["key"], "/foo/x", "") assert.Equal(t, kv0["value"], "XXX", "") assert.Equal(t, kv0["ttl"], 10, "") kv1 := body["kvs"].([]interface{})[1].(map[string]interface{}) assert.Equal(t, kv1["key"], "/foo/y", "") assert.Equal(t, kv1["dir"], true, "") kvs2 := kv1["kvs"].([]interface{})[0].(map[string]interface{}) assert.Equal(t, kvs2["key"], "/foo/y/z", "") assert.Equal(t, kvs2["value"], "YYY", "") }) }
// Ensures that a directory of values can be retrieved for a given key. // // $ curl -X PUT localhost:4001/v1/keys/foo/x -d value=XXX // $ curl -X PUT localhost:4001/v1/keys/foo/y/z -d value=YYY // $ curl localhost:4001/v1/keys/foo // func TestV1GetKeyDir(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") resp, _ := tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo/x"), v) tests.ReadBody(resp) v.Set("value", "YYY") resp, _ = tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo/y/z"), v) tests.ReadBody(resp) resp, _ = tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo")) assert.Equal(t, resp.StatusCode, http.StatusOK) body := tests.ReadBody(resp) nodes := make([]interface{}, 0) if err := json.Unmarshal(body, &nodes); err != nil { panic(fmt.Sprintf("HTTP body JSON parse error: %v", err)) } assert.Equal(t, len(nodes), 2, "") node0 := nodes[0].(map[string]interface{}) assert.Equal(t, node0["action"], "get", "") assert.Equal(t, node0["key"], "/foo/x", "") assert.Equal(t, node0["value"], "XXX", "") node1 := nodes[1].(map[string]interface{}) assert.Equal(t, node1["action"], "get", "") assert.Equal(t, node1["key"], "/foo/y", "") assert.Equal(t, node1["dir"], true, "") }) }
// Ensures that HEAD works. // // $ curl -I localhost:4001/v1/keys/foo/bar -> fail // $ curl -X PUT localhost:4001/v1/keys/foo/bar -d value=XXX // $ curl -I localhost:4001/v1/keys/foo/bar // func TestV1HeadKey(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") fullURL := fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo/bar") resp, _ := tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusNotFound) assert.Equal(t, resp.ContentLength, -1) resp, _ = tests.PutForm(fullURL, v) tests.ReadBody(resp) resp, _ = tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusOK) assert.Equal(t, resp.ContentLength, -1) }) }
// Ensures that a watcher can wait for a value to be set and return it to the client. // // $ curl localhost:4001/v2/keys/foo/bar?wait=true // $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=XXX // func TestV2WatchKey(t *testing.T) { tests.RunServer(func(s *server.Server) { // There exists a little gap between etcd ready to serve and // it actually serves the first request, which means the response // delay could be a little bigger. // This test is time sensitive, so it does one request to ensure // that the server is working. tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar")) var watchResp *http.Response c := make(chan bool) go func() { watchResp, _ = tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar?wait=true")) c <- true }() // Make sure response didn't fire early. time.Sleep(1 * time.Millisecond) // Set a value. v := url.Values{} v.Set("value", "XXX") resp, _ := tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar"), v) tests.ReadBody(resp) // A response should follow from the GET above. time.Sleep(1 * time.Millisecond) select { case <-c: default: t.Fatal("cannot get watch result") } body := tests.ReadBodyJSON(watchResp) assert.NotNil(t, body, "") assert.Equal(t, body["action"], "set", "") node := body["node"].(map[string]interface{}) assert.Equal(t, node["key"], "/foo/bar", "") assert.Equal(t, node["value"], "XXX", "") assert.Equal(t, node["modifiedIndex"], 3, "") }) }
// Ensures that a value can be retrieve for a given key. // // $ curl localhost:4001/v1/keys/foo/bar -> fail // $ curl -X PUT localhost:4001/v1/keys/foo/bar -d value=XXX // $ curl localhost:4001/v1/keys/foo/bar // func TestV1GetKey(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") fullURL := fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo/bar") resp, _ := tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusNotFound) resp, _ = tests.PutForm(fullURL, v) tests.ReadBody(resp) resp, _ = tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusOK) body := tests.ReadBodyJSON(resp) assert.Equal(t, body["action"], "get", "") assert.Equal(t, body["key"], "/foo/bar", "") assert.Equal(t, body["value"], "XXX", "") assert.Equal(t, body["index"], 2, "") }) }
// Ensures that a value can be retrieve for a given key. // // $ curl localhost:4001/v2/keys/foo/bar -> fail // $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=XXX // $ curl localhost:4001/v2/keys/foo/bar // func TestV2GetKey(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") fullURL := fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar") resp, _ := tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusNotFound) resp, _ = tests.PutForm(fullURL, v) tests.ReadBody(resp) resp, _ = tests.Get(fullURL) assert.Equal(t, resp.StatusCode, http.StatusOK) body := tests.ReadBodyJSON(resp) assert.Equal(t, body["action"], "get", "") node := body["node"].(map[string]interface{}) assert.Equal(t, node["key"], "/foo/bar", "") assert.Equal(t, node["value"], "XXX", "") assert.Equal(t, node["modifiedIndex"], 3, "") }) }
// Ensures that a value can be retrieve for a given key. // // $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=XXX // $ curl localhost:4001/v2/keys/foo/bar // func TestV2GetKey(t *testing.T) { tests.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "XXX") resp, _ := tests.PutForm(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar"), v) tests.ReadBody(resp) resp, _ = tests.Get(fmt.Sprintf("http://%s%s", s.URL(), "/v2/keys/foo/bar")) body := tests.ReadBodyJSON(resp) assert.Equal(t, body["action"], "get", "") assert.Equal(t, body["key"], "/foo/bar", "") assert.Equal(t, body["value"], "XXX", "") assert.Equal(t, body["modifiedIndex"], 1, "") }) }
// Ensures that a watcher can wait for a value to be set after a given index. // // $ curl localhost:4001/v2/keys/foo/bar?wait=true&waitIndex=4 // $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=XXX // $ curl -X PUT localhost:4001/v2/keys/foo/bar -d value=YYY // func TestV2WatchKeyWithIndex(t *testing.T) { tests.RunServer(func(s *server.Server) { var body map[string]interface{} c := make(chan bool) go func() { resp, _ := tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar?wait=true&waitIndex=3")) body = tests.ReadBodyJSON(resp) c <- true }() // Make sure response didn't fire early. time.Sleep(1 * time.Millisecond) assert.Nil(t, body, "") // Set a value (before given index). v := url.Values{} v.Set("value", "XXX") resp, _ := tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar"), v) tests.ReadBody(resp) // Make sure response didn't fire early. time.Sleep(1 * time.Millisecond) assert.Nil(t, body, "") // Set a value (before given index). v.Set("value", "YYY") resp, _ = tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/foo/bar"), v) tests.ReadBody(resp) // A response should follow from the GET above. time.Sleep(1 * time.Millisecond) select { case <-c: default: t.Fatal("cannot get watch result") } assert.NotNil(t, body, "") assert.Equal(t, body["action"], "set", "") node := body["node"].(map[string]interface{}) assert.Equal(t, node["key"], "/foo/bar", "") assert.Equal(t, node["value"], "YYY", "") assert.Equal(t, node["modifiedIndex"], 3, "") }) }
// Ensure that the cluster configuration can be updated. func TestClusterConfigSet(t *testing.T) { _, etcds, err := CreateCluster(3, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false) assert.NoError(t, err) defer DestroyCluster(etcds) resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":3, "removeDelay":60}`)) assert.Equal(t, resp.StatusCode, 200) time.Sleep(1 * time.Second) resp, _ = tests.Get("http://localhost:7002/v2/admin/config") body := tests.ReadBodyJSON(resp) assert.Equal(t, resp.StatusCode, 200) assert.Equal(t, body["activeSize"], 3) assert.Equal(t, body["removeDelay"], 60) }
// TestGetMachines tests '/v2/admin/machines' sends back messages of all machines. func TestGetMachines(t *testing.T) { _, etcds, err := CreateCluster(3, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false) assert.NoError(t, err) defer DestroyCluster(etcds) time.Sleep(1 * time.Second) resp, err := tests.Get("http://localhost:7001/v2/admin/machines") if !assert.Equal(t, err, nil) { t.FailNow() } assert.Equal(t, resp.StatusCode, 200) machines := make([]map[string]interface{}, 0) b := tests.ReadBody(resp) json.Unmarshal(b, &machines) assert.Equal(t, len(machines), 3) if machines[0]["state"] != "leader" && machines[1]["state"] != "leader" && machines[2]["state"] != "leader" { t.Errorf("no leader in the cluster") } }
// Ensures that a watcher can wait for a value to be set after a given index. // // $ curl localhost:4001/v2/keys/keyindir/bar?wait=true // $ curl -X PUT localhost:4001/v2/keys/keyindir -d dir=true -d ttl=1 // $ curl -X PUT localhost:4001/v2/keys/keyindir/bar -d value=YYY // func TestV2WatchKeyInDir(t *testing.T) { tests.RunServer(func(s *server.Server) { var body map[string]interface{} c := make(chan bool) // Set a value (before given index). v := url.Values{} v.Set("dir", "true") v.Set("ttl", "1") resp, _ := tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/keyindir"), v) tests.ReadBody(resp) // Set a value (before given index). v = url.Values{} v.Set("value", "XXX") resp, _ = tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/keyindir/bar"), v) tests.ReadBody(resp) go func() { resp, _ := tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/keyindir/bar?wait=true")) body = tests.ReadBodyJSON(resp) c <- true }() // wait for expiration, we do have a up to 500 millisecond delay time.Sleep(1500 * time.Millisecond) select { case <-c: default: t.Fatal("cannot get watch result") } assert.NotNil(t, body, "") assert.Equal(t, body["action"], "expire", "") node := body["node"].(map[string]interface{}) assert.Equal(t, node["key"], "/keyindir", "") }) }
// Ensures that a watcher can wait for a value to be set and return it to the client. // // $ curl localhost:4001/v1/watch/foo/bar // $ curl -X PUT localhost:4001/v1/keys/foo/bar -d value=XXX // func TestV1WatchKey(t *testing.T) { tests.RunServer(func(s *server.Server) { var body map[string]interface{} c := make(chan bool) go func() { resp, _ := tests.Get(fmt.Sprintf("%s%s", s.URL(), "/v1/watch/foo/bar")) body = tests.ReadBodyJSON(resp) c <- true }() // Make sure response didn't fire early. time.Sleep(1 * time.Millisecond) assert.Nil(t, body, "") // Set a value. v := url.Values{} v.Set("value", "XXX") resp, _ := tests.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v1/keys/foo/bar"), v) tests.ReadBody(resp) // A response should follow from the GET above. time.Sleep(1 * time.Millisecond) select { case <-c: default: t.Fatal("cannot get watch result") } assert.NotNil(t, body, "") assert.Equal(t, body["action"], "set", "") assert.Equal(t, body["key"], "/foo/bar", "") assert.Equal(t, body["value"], "XXX", "") assert.Equal(t, body["index"], 2, "") }) }
// Ensure that we can start a v2 node from the log of a v1 node. func TestV1SoloMigration(t *testing.T) { path, _ := ioutil.TempDir("", "etcd-") os.MkdirAll(path, 0777) defer os.RemoveAll(path) nodepath := filepath.Join(path, "node0") fixturepath, _ := filepath.Abs("../fixtures/v1.solo/node0") fmt.Println("DATA_DIR =", nodepath) // Copy over fixture files. c := exec.Command("cp", "-rf", fixturepath, nodepath) if out, err := c.CombinedOutput(); err != nil { fmt.Println(">>>>>>\n", string(out), "<<<<<<") panic("Fixture initialization error:" + err.Error()) } procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} args := []string{"etcd", fmt.Sprintf("-data-dir=%s", nodepath)} args = append(args, "-addr", "127.0.0.1:4001") args = append(args, "-peer-addr", "127.0.0.1:7001") args = append(args, "-name", "node0") process, err := os.StartProcess(EtcdBinPath, args, procAttr) if err != nil { t.Fatal("start process failed:" + err.Error()) return } defer process.Kill() time.Sleep(time.Second) // Ensure deleted message is removed. resp, err := tests.Get("http://localhost:4001/v2/keys/message") tests.ReadBody(resp) assert.Nil(t, err, "") assert.Equal(t, resp.StatusCode, 200, "") }
func testGetLeader(s *server.Server, key string) (string, error) { resp, err := tests.Get(fmt.Sprintf("%s/mod/v2/leader/%s", s.URL(), key)) ret := tests.ReadBody(resp) return string(ret), err }
func testGetLockIndex(s *server.Server, key string) (string, error) { resp, err := tests.Get(fmt.Sprintf("%s/mod/v2/lock/%s?field=index", s.URL(), key)) ret := tests.ReadBody(resp) return string(ret), err }
// Create a full cluster and then change the active size. func TestStandby(t *testing.T) { clusterSize := 15 _, etcds, err := CreateCluster(clusterSize, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false) if !assert.NoError(t, err) { t.Fatal("cannot create cluster") } defer DestroyCluster(etcds) resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"syncInterval":1}`)) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() // Verify that we just have default machines. result, err := c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 9) t.Log("Reconfigure with a smaller active size") resp, _ = tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":7, "syncInterval":1}`)) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } // Wait for two monitor cycles before checking for demotion. time.Sleep((2 * server.ActiveMonitorTimeout) + (2 * time.Second)) // Verify that we now have seven peers. result, err = c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 7) t.Log("Test the functionality of all servers") // Set key. time.Sleep(time.Second) if _, err := c.Set("foo", "bar", 0); err != nil { panic(err) } time.Sleep(time.Second) // Check that all peers and standbys have the value. for i := range etcds { resp, err := tests.Get(fmt.Sprintf("http://localhost:%d/v2/keys/foo", 4000+(i+1))) if assert.NoError(t, err) { body := tests.ReadBodyJSON(resp) if node, _ := body["node"].(map[string]interface{}); assert.NotNil(t, node) { assert.Equal(t, node["value"], "bar") } } } t.Log("Reconfigure with larger active size and wait for join") resp, _ = tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":8, "syncInterval":1}`)) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } time.Sleep((1 * time.Second) + (1 * time.Second)) // Verify that exactly eight machines are in the cluster. result, err = c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 8) }
// Create a full cluster and then change the active size dramatically. func TestStandbyDramaticChange(t *testing.T) { clusterSize := 9 _, etcds, err := CreateCluster(clusterSize, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false) assert.NoError(t, err) defer DestroyCluster(etcds) if err != nil { t.Fatal("cannot create cluster") } time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() num := clusterSize for i := 0; i < 3; i++ { for inc := 0; inc < 2; inc++ { // Verify that we just have i machines. result, err := c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), num) if inc == 0 { num -= 6 } else { num += 6 } t.Log("Reconfigure with active size", num) resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(fmt.Sprintf(`{"activeSize":%d, "syncInterval":1}`, num))) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } if inc == 0 { // Wait for monitor cycles before checking for demotion. time.Sleep(6*server.ActiveMonitorTimeout + (1 * time.Second)) } else { time.Sleep(time.Second + (1 * time.Second)) } // Verify that we now have peers. result, err = c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), num) t.Log("Test the functionality of all servers") // Set key. if _, err := c.Set("foo", "bar", 0); err != nil { panic(err) } time.Sleep(100 * time.Millisecond) // Check that all peers and standbys have the value. for i := range etcds { resp, err := tests.Get(fmt.Sprintf("http://localhost:%d/v2/keys/foo", 4000+(i+1))) if assert.NoError(t, err) { body := tests.ReadBodyJSON(resp) if node, _ := body["node"].(map[string]interface{}); assert.NotNil(t, node) { assert.Equal(t, node["value"], "bar") } } } } } }
func testGetLockValue(s *server.Server, key string) (string, int, error) { resp, err := tests.Get(fmt.Sprintf("%s/mod/v2/lock/%s", s.URL(), key)) ret := tests.ReadBody(resp) return string(ret), resp.StatusCode, err }
// Create a full cluster and then add extra an extra standby node. func TestStandby(t *testing.T) { t.Skip("functionality unimplemented") clusterSize := 10 // DefaultActiveSize + 1 _, etcds, err := CreateCluster(clusterSize, &os.ProcAttr{Files: []*os.File{nil, os.Stdout, os.Stderr}}, false) assert.NoError(t, err) defer DestroyCluster(etcds) if err != nil { t.Fatal("cannot create cluster") } c := etcd.NewClient(nil) c.SyncCluster() // Set key. time.Sleep(time.Second) if _, err := c.Set("foo", "bar", 0); err != nil { panic(err) } time.Sleep(time.Second) // Check that all peers and standbys have the value. for i := range etcds { resp, err := tests.Get(fmt.Sprintf("http://localhost:%d/v2/keys/foo", 4000+(i+1))) if assert.NoError(t, err) { body := tests.ReadBodyJSON(resp) if node, _ := body["node"].(map[string]interface{}); assert.NotNil(t, node) { assert.Equal(t, node["value"], "bar") } } } // Verify that we have one standby. result, err := c.Get("_etcd/standbys", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 1) // Reconfigure with larger active size (10 nodes) and wait for promotion. resp, _ := tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":10, "promoteDelay":1800}`)) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } time.Sleep(server.ActiveMonitorTimeout + (1 * time.Second)) // Verify that the standby node is now a peer. result, err = c.Get("_etcd/standbys", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 0) // Reconfigure with a smaller active size (8 nodes). resp, _ = tests.Put("http://localhost:7001/v2/admin/config", "application/json", bytes.NewBufferString(`{"activeSize":8, "promoteDelay":1800}`)) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } // Wait for two monitor cycles before checking for demotion. time.Sleep((2 * server.ActiveMonitorTimeout) + (1 * time.Second)) // Verify that we now have eight peers. result, err = c.Get("_etcd/machines", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 8) // Verify that we now have two standbys. result, err = c.Get("_etcd/standbys", false, true) assert.NoError(t, err) assert.Equal(t, len(result.Node.Nodes), 2) }
// Create a five nodes // Kill all the nodes and restart, then remove the leader func TestMultiNodeKillAllAndRecoveryAndRemoveLeader(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} stop := make(chan bool) leaderChan := make(chan string, 1) all := make(chan bool, 1) clusterSize := 5 argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false) defer DestroyCluster(etcds) if err != nil { t.Fatal("cannot create cluster") } c := etcd.NewClient(nil) go Monitor(clusterSize, clusterSize, leaderChan, all, stop) <-all <-leaderChan stop <- true // It needs some time to sync current commits and write it to disk. // Or some instance may be restarted as a new peer, and we don't support // to connect back the old cluster that doesn't have majority alive // without log now. time.Sleep(time.Second) c.SyncCluster() // kill all DestroyCluster(etcds) time.Sleep(time.Second) stop = make(chan bool) leaderChan = make(chan string, 1) all = make(chan bool, 1) time.Sleep(time.Second) for i := 0; i < clusterSize; i++ { etcds[i], err = os.StartProcess(EtcdBinPath, argGroup[i], procAttr) } go Monitor(clusterSize, 1, leaderChan, all, stop) <-all leader := <-leaderChan _, err = c.Set("foo", "bar", 0) if err != nil { t.Fatalf("Recovery error: %s", err) } port, _ := strconv.Atoi(strings.Split(leader, ":")[2]) num := port - 7000 resp, _ := tests.Delete(leader+"/v2/admin/machines/node"+strconv.Itoa(num), "application/json", nil) if !assert.Equal(t, resp.StatusCode, 200) { t.FailNow() } // check the old leader is in standby mode now time.Sleep(time.Second) resp, _ = tests.Get(leader + "/name") assert.Equal(t, resp.StatusCode, 404) }