// run clustering based tests
func ClusterTests(c platform.TestCluster) error {
if plog.LevelAt(capnslog.DEBUG) {
// get journalctl -f from all machines before starting
for _, m := range c.Machines() {
if err := platform.StreamJournal(m); err != nil {
return fmt.Errorf("failed to start journal: %v", err)
}
}
}
// wait for etcd to come up
if err := etcd.GetClusterHealth(c.Machines()[0], len(c.Machines())); err != nil {
return err
}
tests := c.ListNativeFunctions()
for _, name := range tests {
plog.Noticef("running %v...", name)
err := c.RunNative(name, c.Machines()[0])
if err != nil {
return err
}
}
return nil
}
func discovery(cluster platform.Cluster, version int) error {
if plog.LevelAt(capnslog.DEBUG) {
// get journalctl -f from all machines before starting
for _, m := range cluster.Machines() {
if err := platform.StreamJournal(m); err != nil {
return fmt.Errorf("failed to start journal: %v", err)
}
}
}
// start etcd on each machine asynchronously.
for _, m := range cluster.Machines() {
if err := doStart(m, version, false); err != nil {
return err
}
}
// block until each instance is reported as started.
for i, m := range cluster.Machines() {
if err := doStart(m, version, true); err != nil {
return err
}
plog.Infof("etcd instance%d started", i)
}
var keyMap map[string]string
var retryFuncs []func() error
retryFuncs = append(retryFuncs, func() error {
var err error
keyMap, err = SetKeys(cluster, 5)
if err != nil {
return err
}
return nil
})
retryFuncs = append(retryFuncs, func() error {
var quorumRead bool
if version == 2 {
quorumRead = true
}
if err := CheckKeys(cluster, keyMap, quorumRead); err != nil {
return err
}
return nil
})
for _, retry := range retryFuncs {
if err := util.Retry(5, 5*time.Second, retry); err != nil {
return fmt.Errorf("discovery failed health check: %v", err)
}
// NOTE(pb): etcd1 seems to fail in an odd way when I try quorum
// read, instead just sleep between setting and getting.
time.Sleep(2 * time.Second)
}
return nil
}
// run clustering based tests
func ClusterTests(c platform.TestCluster) error {
if plog.LevelAt(capnslog.DEBUG) {
// get journalctl -f from all machines before starting
for _, m := range c.Machines() {
if err := platform.StreamJournal(m); err != nil {
return fmt.Errorf("failed to start journal: %v", err)
}
}
}
// make sure etcd is up and running
var keyMap map[string]string
var retryFuncs []func() error
retryFuncs = append(retryFuncs, func() error {
var err error
keyMap, err = etcd.SetKeys(c, 3)
if err != nil {
return err
}
return nil
})
retryFuncs = append(retryFuncs, func() error {
if err := etcd.CheckKeys(c, keyMap, true); err != nil {
return err
}
return nil
})
for _, retry := range retryFuncs {
if err := util.Retry(5, 5*time.Second, retry); err != nil {
return fmt.Errorf("etcd failed health check: %v", err)
}
}
tests := c.ListNativeFunctions()
for _, name := range tests {
plog.Noticef("running %v...", name)
err := c.RunNative(name, c.Machines()[0])
if err != nil {
return err
}
}
return nil
}
func RollingUpgrade(cluster platform.TestCluster) error {
var (
firstVersion = cluster.Options["EtcdUpgradeVersion"]
secondVersion = cluster.Options["EtcdUpgradeVersion2"]
firstBin = cluster.Options["EtcdUpgradeBin"]
secondBin = cluster.Options["EtcdUpgradeBin2"]
)
csize := len(cluster.Machines())
if plog.LevelAt(capnslog.DEBUG) {
// get journalctl -f from all machines before starting
for _, m := range cluster.Machines() {
if err := platform.StreamJournal(m); err != nil {
return fmt.Errorf("failed to start journal: %v", err)
}
}
}
// drop in starting etcd binary
plog.Debug("adding files to cluster")
if err := cluster.DropFile(firstBin); err != nil {
return err
}
// drop in etcd binary to upgrade to
if err := cluster.DropFile(secondBin); err != nil {
return err
}
// replace existing etcd2 binary with 2.0.12
plog.Info("replacing etcd with 2.0.12")
firstPath := filepath.Join(dropPath, filepath.Base(firstBin))
for _, m := range cluster.Machines() {
if err := replaceEtcd2Bin(m, firstPath); err != nil {
return err
}
}
// start 2.0 cluster
plog.Info("starting 2.0 cluster")
for _, m := range cluster.Machines() {
if err := startEtcd2(m); err != nil {
return err
}
}
for _, m := range cluster.Machines() {
if err := getClusterHealth(m, csize); err != nil {
return err
}
}
if firstVersion != "" {
for _, m := range cluster.Machines() {
if err := checkEtcdVersion(cluster, m, firstVersion); err != nil {
return err
}
}
}
// set some values on all nodes
mapSet, err := SetKeys(cluster, settingSize)
if err != nil {
return err
}
// rolling replacement checking cluster health, and
// version after each replaced binary. Also test
plog.Info("rolling upgrade to 2.1")
secondPath := filepath.Join(dropPath, filepath.Base(secondBin))
for i, m := range cluster.Machines() {
// check current value set
if err := CheckKeys(cluster, mapSet, true); err != nil {
return err
}
plog.Infof("stopping instance %v", i)
if err := stopEtcd2(m); err != nil {
return err
}
if err := replaceEtcd2Bin(m, secondPath); err != nil {
return err
}
// set some values while running down a node and update set
tempSet, err := SetKeys(cluster, settingSize)
if err != nil {
return err
}
mapCopy(mapSet, tempSet)
plog.Infof("starting instance %v with upgraded binary", i)
if err := startEtcd2(m); err != nil {
return err
}
for _, m := range cluster.Machines() {
if err := getClusterHealth(m, csize); err != nil {
return err
}
}
}
// set some more values
tempSet, err := SetKeys(cluster, settingSize)
if err != nil {
return err
}
mapCopy(mapSet, tempSet)
// final check all values written correctly
if err := CheckKeys(cluster, mapSet, true); err != nil {
return err
}
// check version is now 2.1
if secondVersion != "" {
for _, m := range cluster.Machines() {
if err := checkEtcdVersion(cluster, m, secondVersion); err != nil {
return err
}
}
}
return nil
}