func CommandFunc(cmd *cobra.Command, args []string) { defer func() { fmt.Println("deleting...") os.RemoveAll("infra1.etcd") os.RemoveAll("infra2.etcd") os.RemoveAll("infra3.etcd") }() oldCmds := make([]*exec.Cmd, 3) oldOutputs := make([]io.ReadCloser, 3) newCmds := make([]*exec.Cmd, 3) newOutputs := make([]io.ReadCloser, 3) for i := range oldCmds { oldCmd := exec.Command(cmdFlag.EtcdOld, getInfraFlags(i+1)...) oldCmds[i] = oldCmd oldOutput, err := oldCmd.StderrPipe() if err != nil { fmt.Fprintln(os.Stderr, err) return } oldOutputs[i] = oldOutput newCmd := exec.Command(cmdFlag.EtcdNew, getInfraFlags(i+1)...) newCmds[i] = newCmd newOutput, err := newCmd.StderrPipe() if err != nil { fmt.Fprintln(os.Stderr, err) return } newOutputs[i] = newOutput } errChan := make(chan error) done := make(chan struct{}) for i := range oldCmds { cmd := oldCmds[i] go func(i int, cmd *exec.Cmd) { if err := cmd.Start(); err != nil { errChan <- err return } done <- struct{}{} }(i, cmd) } cn := 0 for cn != 3 { cn++ select { case err := <-errChan: fmt.Fprintln(os.Stderr, err) return case <-done: } } becameActiveCount := 0 for i, o := range oldOutputs { go func(i int, reader io.ReadCloser) { scanner := bufio.NewScanner(reader) for { for scanner.Scan() { txt := scanner.Text() fmt.Printf("[old infra%d] %s\n", i+1, txt) if strings.Contains(txt, memberStartReadyString) { mu.Lock() nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE" mu.Unlock() fmt.Printf("[old infra%d] %s READY!!!!!!!!!!!!!\n", i+1, txt) done <- struct{}{} } if strings.HasSuffix(txt, memberReStartReadySuffix) { fmt.Printf("[old infra%d] reconnected!\n", i+1) mu.Lock() nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE" mu.Unlock() becameActiveCount++ } } } if err := scanner.Err(); err != nil { errChan <- err return } }(i, o) } for i, o := range newOutputs { go func(i int, reader io.ReadCloser) { scanner := bufio.NewScanner(reader) for { for scanner.Scan() { txt := scanner.Text() fmt.Printf("[new infra%d] %s\n", i+1, txt) if strings.HasSuffix(txt, memberReStartReadySuffix) { fmt.Printf("[new infra%d] reconnected!\n", i+1) mu.Lock() nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE" mu.Unlock() becameActiveCount++ } } } if err := scanner.Err(); err != nil { errChan <- err return } }(i, o) } cn = 0 for cn != 3 { cn++ select { case err := <-errChan: fmt.Fprintln(os.Stderr, err) return case <-done: } } es := stress(10) if es != nil { log.Println(es) return } go func() { es := stress(50) if es != nil { log.Println(es) return } }() for i := 0; i < 3; i++ { fmt.Printf("[old infra%d] killing...\n", i+1) mu.Lock() nodeStatus[fmt.Sprintf("infra%d", i+1)] = "KILLED" mu.Unlock() if err := syscall.Kill(oldCmds[i].Process.Pid, syscall.SIGKILL); err != nil { fmt.Fprintln(os.Stderr, err) return } fmt.Printf("[old infra%d] killed!\n", i+1) time.Sleep(10 * time.Second) fmt.Printf("[new infra%d] restarting...\n", i+1) if err := newCmds[i].Start(); err != nil { fmt.Fprintln(os.Stderr, err) return } mu.Lock() nodeStatus[fmt.Sprintf("infra%d", i+1)] = "LIVE" mu.Unlock() fmt.Printf("[new infra%d] restarted!\n", i+1) time.Sleep(10 * time.Second) } // 6(2 per node) at the beginning of cluster + 12(4 per kill) during migration = 18 if becameActiveCount >= 18 { fmt.Printf("migration successful from %s to %s (node status %v)\n", cmdFlag.EtcdOld, cmdFlag.EtcdNew, nodeStatus) } else { fmt.Printf("migration failed from %s to %s (becameActiveCount %d, node status %v)\n", cmdFlag.EtcdOld, cmdFlag.EtcdNew, becameActiveCount, nodeStatus) } }