// TestDiscoverySecondPeerUp ensures that a second peer joining a discovery // cluster works. func TestDiscoverySecondPeerUp(t *testing.T) { etcdtest.RunServer(func(s *server.Server) { v := url.Values{} v.Set("value", "started") resp, err := etcdtest.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/_etcd/registry/3/_state"), v) assert.Equal(t, resp.StatusCode, http.StatusCreated) u, ok := s.PeerURL("ETCDTEST") if !ok { t.Fatalf("Couldn't find the URL") } wc := goetcd.NewClient([]string{s.URL()}) testResp, err := wc.Set("test", "0", 0) if err != nil { t.Fatalf("Couldn't set a test key on the leader %v", err) } v = url.Values{} v.Set("value", u) resp, err = etcdtest.PutForm(fmt.Sprintf("%s%s", s.URL(), "/v2/keys/_etcd/registry/3/ETCDTEST"), v) assert.Equal(t, resp.StatusCode, http.StatusCreated) proc, err := startServer([]string{"-discovery", s.URL() + "/v2/keys/_etcd/registry/3"}) if err != nil { t.Fatal(err.Error()) } defer stopServer(proc) watch := fmt.Sprintf("%s%s%d", s.URL(), "/v2/keys/_etcd/registry/3/node1?wait=true&waitIndex=", testResp.EtcdIndex) resp, err = http.Get(watch) if err != nil { t.Fatal(err.Error()) } // TODO(bp): need to have a better way of knowing a machine is up for i := 0; i < 10; i++ { time.Sleep(1 * time.Second) etcdc := goetcd.NewClient(nil) _, err = etcdc.Set("foobar", "baz", 0) if err == nil { break } } if err != nil { t.Fatal(err.Error()) } }) }
// Sending set commands func Set(stop chan bool) { stopSet := false i := 0 c := etcd.NewClient(nil) for { key := fmt.Sprintf("%s_%v", "foo", i) result, err := c.Set(key, "bar", 0) if err != nil || result.Node.Key != "/"+key || result.Node.Value != "bar" { select { case <-stop: stopSet = true default: } } select { case <-stop: stopSet = true default: } if stopSet { break } i++ } stop <- true }
// Create a five nodes // Kill all the nodes and restart func TestMultiNodeKillAllAndRecovery(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} clusterSize := 5 argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false) defer DestroyCluster(etcds) if err != nil { t.Fatal("cannot create cluster") } c := etcd.NewClient(nil) c.SyncCluster() time.Sleep(time.Second) // send 10 commands for i := 0; i < 10; i++ { // Test Set _, err := c.Set("foo", "bar", 0) if err != nil { panic(err) } } time.Sleep(time.Second) // kill all DestroyCluster(etcds) time.Sleep(time.Second) stop := make(chan bool) leaderChan := make(chan string, 1) all := make(chan bool, 1) time.Sleep(time.Second) for i := 0; i < clusterSize; i++ { etcds[i], err = os.StartProcess(EtcdBinPath, argGroup[i], procAttr) } go Monitor(clusterSize, 1, leaderChan, all, stop) <-all <-leaderChan result, err := c.Set("foo", "bar", 0) if err != nil { t.Fatalf("Recovery error: %s", err) } if result.Node.ModifiedIndex != 16 { t.Fatalf("recovery failed! [%d/16]", result.Node.ModifiedIndex) } }
func (d *Discoverer) Do(discoveryURL string, name string, peer string) (peers []string, err error) { d.name = name d.peer = peer d.discoveryURL = discoveryURL u, err := url.Parse(discoveryURL) if err != nil { return } // prefix is prepended to all keys for this discovery d.prefix = strings.TrimPrefix(u.Path, "/v2/keys/") // keep the old path in case we need to set the KeyPrefix below oldPath := u.Path u.Path = "" // Connect to a scheme://host not a full URL with path log.Infof("Discovery via %s using prefix %s.", u.String(), d.prefix) d.client = etcd.NewClient([]string{u.String()}) if !strings.HasPrefix(oldPath, "/v2/keys") { d.client.SetKeyPrefix("") } // Register this machine first and announce that we are a member of // this cluster err = d.heartbeat() if err != nil { return } // Start the very slow heartbeat to the cluster now in anticipation // that everything is going to go alright now go d.startHeartbeat() // Attempt to take the leadership role, if there is no error we are it! resp, err := d.client.Create(path.Join(d.prefix, stateKey), startedState, 0) // Bail out on unexpected errors if err != nil { if clientErr, ok := err.(*etcd.EtcdError); !ok || clientErr.ErrorCode != etcdErr.EcodeNodeExist { return nil, err } } // If we got a response then the CAS was successful, we are leader if resp != nil && resp.Node.Value == startedState { // We are the leader, we have no peers log.Infof("Discovery _state was empty, so this machine is the initial leader.") return nil, nil } // Fall through to finding the other discovery peers return d.findPeers() }
// This test creates a single node and then set a value to it. // Then this test kills the node and restart it and tries to get the value again. func TestSingleNodeRecovery(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} args := []string{"etcd", "-name=node1", "-data-dir=/tmp/node1"} process, err := os.StartProcess(EtcdBinPath, append(args, "-f"), procAttr) if err != nil { t.Fatal("start process failed:" + err.Error()) return } time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() // Test Set result, err := c.Set("foo", "bar", 100) node := result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) process.Kill() process, err = os.StartProcess(EtcdBinPath, args, procAttr) defer process.Kill() if err != nil { t.Fatal("start process failed:" + err.Error()) return } time.Sleep(time.Second) result, err = c.Get("foo", false, false) node = result.Node if err != nil { t.Fatal("get fail: " + err.Error()) return } if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL > 99 { if err != nil { t.Fatal(err) } t.Fatalf("Recovery Get failed with %s %s %v", node.Key, node.Value, node.TTL) } }
func write(endpoint string, requests int, end chan int) { client := etcd.NewClient([]string{endpoint}) for i := 0; i < requests; i++ { key := strconv.Itoa(i) _, err := client.Set(key, key, 0) if err != nil { println(err.Error()) } } end <- 1 }
// NewHandler creates an HTTP handler that can be registered on a router. func NewHandler(addr string) http.Handler { h := &handler{ Router: mux.NewRouter(), client: etcd.NewClient([]string{addr}), } h.StrictSlash(false) h.handleFunc("/{key:.*}", h.getIndexHandler).Methods("GET") h.handleFunc("/{key:.*}", h.acquireHandler).Methods("POST") h.handleFunc("/{key:.*}", h.renewLockHandler).Methods("PUT") h.handleFunc("/{key:.*}", h.releaseLockHandler).Methods("DELETE") return h }
func watch(endpoint string, key string) { client := etcd.NewClient([]string{endpoint}) receiver := make(chan *etcd.Response) go client.Watch(key, 0, true, receiver, nil) log.Printf("watching: %s", key) received := 0 for { <-receiver received++ } }
// Create a three nodes and try to set value func templateTestSimpleMultiNode(t *testing.T, tls bool) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} clusterSize := 3 _, etcds, err := CreateCluster(clusterSize, procAttr, tls) if err != nil { t.Fatal("cannot create cluster") } defer DestroyCluster(etcds) time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() // Test Set result, err := c.Set("foo", "bar", 100) node := result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) result, err = c.Set("foo", "bar", 100) node = result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } t.Fatalf("Set 2 failed with %s %s %v", node.Key, node.Value, node.TTL) } }
// Create a five nodes // Randomly kill one of the node and keep on sending set command to the cluster func TestMultiNodeKillOne(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} clusterSize := 5 argGroup, etcds, err := CreateCluster(clusterSize, procAttr, false) if err != nil { t.Fatal("cannot create cluster") } defer DestroyCluster(etcds) time.Sleep(2 * time.Second) c := etcd.NewClient(nil) c.SyncCluster() stop := make(chan bool) // Test Set go Set(stop) for i := 0; i < 10; i++ { num := rand.Int() % clusterSize fmt.Println("kill node", num+1) // kill etcds[num].Kill() etcds[num].Release() time.Sleep(time.Second) // restart etcds[num], err = os.StartProcess(EtcdBinPath, argGroup[num], procAttr) if err != nil { panic(err) } time.Sleep(time.Second) } fmt.Println("stop") stop <- true <-stop }
// remove the node and node rejoin with previous log func TestRemoveNode(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} clusterSize := 3 argGroup, etcds, _ := CreateCluster(clusterSize, procAttr, false) defer DestroyCluster(etcds) time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() rmReq, _ := http.NewRequest("DELETE", "http://127.0.0.1:7001/remove/node3", nil) client := &http.Client{} for i := 0; i < 2; i++ { for i := 0; i < 2; i++ { client.Do(rmReq) fmt.Println("send remove to node3 and wait for its exiting") etcds[2].Wait() resp, err := c.Get("_etcd/machines", false, false) if err != nil { panic(err) } if len(resp.Node.Nodes) != 2 { t.Fatal("cannot remove peer") } if i == 1 { // rejoin with log etcds[2], err = os.StartProcess(EtcdBinPath, argGroup[2], procAttr) } else { // rejoin without log etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr) } if err != nil { panic(err) } time.Sleep(time.Second) resp, err = c.Get("_etcd/machines", false, false) if err != nil { panic(err) } if len(resp.Node.Nodes) != 3 { t.Fatalf("add peer fails #1 (%d != 3)", len(resp.Node.Nodes)) } } // first kill the node, then remove it, then add it back for i := 0; i < 2; i++ { etcds[2].Kill() fmt.Println("kill node3 and wait for its exiting") etcds[2].Wait() client.Do(rmReq) resp, err := c.Get("_etcd/machines", false, false) if err != nil { panic(err) } if len(resp.Node.Nodes) != 2 { t.Fatal("cannot remove peer") } if i == 1 { // rejoin with log etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2]), procAttr) } else { // rejoin without log etcds[2], err = os.StartProcess(EtcdBinPath, append(argGroup[2], "-f"), procAttr) } if err != nil { panic(err) } time.Sleep(time.Second) resp, err = c.Get("_etcd/machines", false, false) if err != nil { panic(err) } if len(resp.Node.Nodes) != 3 { t.Fatalf("add peer fails #2 (%d != 3)", len(resp.Node.Nodes)) } } } }
// Create a single node and try to set value func TestSingleNode(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} args := []string{"etcd", "-name=node1", "-f", "-data-dir=/tmp/node1"} process, err := os.StartProcess(EtcdBinPath, args, procAttr) if err != nil { t.Fatal("start process failed:" + err.Error()) return } defer process.Kill() time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() // Test Set result, err := c.Set("foo", "bar", 100) node := result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal("Set 1: ", err) } t.Fatalf("Set 1 failed with %s %s %v", node.Key, node.Value, node.TTL) } time.Sleep(time.Second) result, err = c.Set("foo", "bar", 100) node = result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL != 100 { if err != nil { t.Fatal("Set 2: ", err) } t.Fatalf("Set 2 failed with %s %s %v", node.Key, node.Value, node.TTL) } // Add a test-and-set test // First, we'll test we can change the value if we get it write result, err = c.CompareAndSwap("foo", "foobar", 100, "bar", 0) node = result.Node if err != nil || node.Key != "/foo" || node.Value != "foobar" || node.TTL != 100 { if err != nil { t.Fatal(err) } t.Fatalf("Set 3 failed with %s %s %v", node.Key, node.Value, node.TTL) } // Next, we'll make sure we can't set it without the correct prior value _, err = c.CompareAndSwap("foo", "foofoo", 100, "bar", 0) if err == nil { t.Fatalf("Set 4 expecting error when setting key with incorrect previous value") } }
// This test creates a single node and then set a value to it to trigger snapshot func TestSimpleSnapshot(t *testing.T) { procAttr := new(os.ProcAttr) procAttr.Files = []*os.File{nil, os.Stdout, os.Stderr} args := []string{"etcd", "-name=node1", "-data-dir=/tmp/node1", "-snapshot=true", "-snapshot-count=500"} process, err := os.StartProcess(EtcdBinPath, append(args, "-f"), procAttr) if err != nil { t.Fatal("start process failed:" + err.Error()) } defer process.Kill() time.Sleep(time.Second) c := etcd.NewClient(nil) c.SyncCluster() // issue first 501 commands for i := 0; i < 501; i++ { result, err := c.Set("foo", "bar", 100) node := result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } t.Fatalf("Set failed with %s %s %v", node.Key, node.Value, node.TTL) } } // wait for a snapshot interval time.Sleep(3 * time.Second) snapshots, err := ioutil.ReadDir("/tmp/node1/snapshot") if err != nil { t.Fatal("list snapshot failed:" + err.Error()) } if len(snapshots) != 1 { t.Fatal("wrong number of snapshot :[1/", len(snapshots), "]") } index, _ := strconv.Atoi(snapshots[0].Name()[2:5]) if index < 507 || index > 510 { t.Fatal("wrong name of snapshot :", snapshots[0].Name()) } // issue second 501 commands for i := 0; i < 501; i++ { result, err := c.Set("foo", "bar", 100) node := result.Node if err != nil || node.Key != "/foo" || node.Value != "bar" || node.TTL < 95 { if err != nil { t.Fatal(err) } t.Fatalf("Set failed with %s %s %v", node.Key, node.Value, node.TTL) } } // wait for a snapshot interval time.Sleep(3 * time.Second) snapshots, err = ioutil.ReadDir("/tmp/node1/snapshot") if err != nil { t.Fatal("list snapshot failed:" + err.Error()) } if len(snapshots) != 1 { t.Fatal("wrong number of snapshot :[1/", len(snapshots), "]") } index, _ = strconv.Atoi(snapshots[0].Name()[2:6]) if index < 1014 || index > 1017 { t.Fatal("wrong name of snapshot :", snapshots[0].Name()) } }