// run clustering based tests func ClusterTests(c platform.TestCluster) error { if plog.LevelAt(capnslog.DEBUG) { // get journalctl -f from all machines before starting for _, m := range c.Machines() { if err := platform.StreamJournal(m); err != nil { return fmt.Errorf("failed to start journal: %v", err) } } } // wait for etcd to come up if err := etcd.GetClusterHealth(c.Machines()[0], len(c.Machines())); err != nil { return err } tests := c.ListNativeFunctions() for _, name := range tests { plog.Noticef("running %v...", name) err := c.RunNative(name, c.Machines()[0]) if err != nil { return err } } return nil }
func discovery(cluster platform.Cluster, version int) error { if plog.LevelAt(capnslog.DEBUG) { // get journalctl -f from all machines before starting for _, m := range cluster.Machines() { if err := platform.StreamJournal(m); err != nil { return fmt.Errorf("failed to start journal: %v", err) } } } // start etcd on each machine asynchronously. for _, m := range cluster.Machines() { if err := doStart(m, version, false); err != nil { return err } } // block until each instance is reported as started. for i, m := range cluster.Machines() { if err := doStart(m, version, true); err != nil { return err } plog.Infof("etcd instance%d started", i) } var keyMap map[string]string var retryFuncs []func() error retryFuncs = append(retryFuncs, func() error { var err error keyMap, err = SetKeys(cluster, 5) if err != nil { return err } return nil }) retryFuncs = append(retryFuncs, func() error { var quorumRead bool if version == 2 { quorumRead = true } if err := CheckKeys(cluster, keyMap, quorumRead); err != nil { return err } return nil }) for _, retry := range retryFuncs { if err := util.Retry(5, 5*time.Second, retry); err != nil { return fmt.Errorf("discovery failed health check: %v", err) } // NOTE(pb): etcd1 seems to fail in an odd way when I try quorum // read, instead just sleep between setting and getting. time.Sleep(2 * time.Second) } return nil }
// run clustering based tests func ClusterTests(c platform.TestCluster) error { if plog.LevelAt(capnslog.DEBUG) { // get journalctl -f from all machines before starting for _, m := range c.Machines() { if err := platform.StreamJournal(m); err != nil { return fmt.Errorf("failed to start journal: %v", err) } } } // make sure etcd is up and running var keyMap map[string]string var retryFuncs []func() error retryFuncs = append(retryFuncs, func() error { var err error keyMap, err = etcd.SetKeys(c, 3) if err != nil { return err } return nil }) retryFuncs = append(retryFuncs, func() error { if err := etcd.CheckKeys(c, keyMap, true); err != nil { return err } return nil }) for _, retry := range retryFuncs { if err := util.Retry(5, 5*time.Second, retry); err != nil { return fmt.Errorf("etcd failed health check: %v", err) } } tests := c.ListNativeFunctions() for _, name := range tests { plog.Noticef("running %v...", name) err := c.RunNative(name, c.Machines()[0]) if err != nil { return err } } return nil }
func RollingUpgrade(cluster platform.TestCluster) error { var ( firstVersion = cluster.Options["EtcdUpgradeVersion"] secondVersion = cluster.Options["EtcdUpgradeVersion2"] firstBin = cluster.Options["EtcdUpgradeBin"] secondBin = cluster.Options["EtcdUpgradeBin2"] ) csize := len(cluster.Machines()) if plog.LevelAt(capnslog.DEBUG) { // get journalctl -f from all machines before starting for _, m := range cluster.Machines() { if err := platform.StreamJournal(m); err != nil { return fmt.Errorf("failed to start journal: %v", err) } } } // drop in starting etcd binary plog.Debug("adding files to cluster") if err := cluster.DropFile(firstBin); err != nil { return err } // drop in etcd binary to upgrade to if err := cluster.DropFile(secondBin); err != nil { return err } // replace existing etcd2 binary with 2.0.12 plog.Info("replacing etcd with 2.0.12") firstPath := filepath.Join(dropPath, filepath.Base(firstBin)) for _, m := range cluster.Machines() { if err := replaceEtcd2Bin(m, firstPath); err != nil { return err } } // start 2.0 cluster plog.Info("starting 2.0 cluster") for _, m := range cluster.Machines() { if err := startEtcd2(m); err != nil { return err } } for _, m := range cluster.Machines() { if err := getClusterHealth(m, csize); err != nil { return err } } if firstVersion != "" { for _, m := range cluster.Machines() { if err := checkEtcdVersion(cluster, m, firstVersion); err != nil { return err } } } // set some values on all nodes mapSet, err := SetKeys(cluster, settingSize) if err != nil { return err } // rolling replacement checking cluster health, and // version after each replaced binary. Also test plog.Info("rolling upgrade to 2.1") secondPath := filepath.Join(dropPath, filepath.Base(secondBin)) for i, m := range cluster.Machines() { // check current value set if err := CheckKeys(cluster, mapSet, true); err != nil { return err } plog.Infof("stopping instance %v", i) if err := stopEtcd2(m); err != nil { return err } if err := replaceEtcd2Bin(m, secondPath); err != nil { return err } // set some values while running down a node and update set tempSet, err := SetKeys(cluster, settingSize) if err != nil { return err } mapCopy(mapSet, tempSet) plog.Infof("starting instance %v with upgraded binary", i) if err := startEtcd2(m); err != nil { return err } for _, m := range cluster.Machines() { if err := getClusterHealth(m, csize); err != nil { return err } } } // set some more values tempSet, err := SetKeys(cluster, settingSize) if err != nil { return err } mapCopy(mapSet, tempSet) // final check all values written correctly if err := CheckKeys(cluster, mapSet, true); err != nil { return err } // check version is now 2.1 if secondVersion != "" { for _, m := range cluster.Machines() { if err := checkEtcdVersion(cluster, m, secondVersion); err != nil { return err } } } return nil }